Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[EEG] Database Architecture for HED Tags #8036

Merged
merged 7 commits into from
Jun 28, 2022
Merged

[EEG] Database Architecture for HED Tags #8036

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
8cbc947
DB tables for HED Tags
Mar 3, 2022
82112f4
viz bugfixes events file access
Mar 7, 2022
0d6cef9
make param description nullable
jesscall Mar 9, 2022
aa07a6d
move AssembledHED to physiological_task_event
May 17, 2022
19d70cb
phpcs
Jun 14, 2022
cc07a82
use ImagingFileTypes table
jesscall Jun 21, 2022
1d83aca
use ImagingFileTypes for physio event files
jesscall Jun 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion SQL/0000-00-00-schema.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1715,7 +1715,8 @@ INSERT INTO `parameter_type` (Name, Alias, Type, Description, SourceFrom) VALUES
('fov_dimensions','FieldOfViewDimensions','text','Dimensions of the field of view, in mm. If Field of View Shape (0018,1147) is: RECTANGLE: row dimension followed by column. ROUND: diameter. HEXAGONAL: diameter of a circumscribed circle. DICOM:0018_1149','parameter_file'),
('laterality','Laterality','text','Laterality of (paired) body part examined. Required if the body part examined is a paired structure and Image Laterality (0020,0062) or Frame Laterality (0020,9072) are not sent. DICOM:0020_0060','parameter_file'),
('position_reference_indicator','PositionReferenceIndicator','text','Part of the imaging target used as a reference. DICOM:0020_1040','parameter_file'),
('pixel_padding_value','PixelPaddingValue','text','Value of pixels added to non-rectangular image to pad to rectangular format. DICOM:0028_0120','parameter_file');
('pixel_padding_value','PixelPaddingValue','text','Value of pixels added to non-rectangular image to pad to rectangular format. DICOM:0028_0120','parameter_file'),
('HEDVersion', 'HEDVersion', 'text', 'HED Schema Version','physiological_parameter_file');

CREATE TABLE `parameter_type_category` (
`ParameterTypeCategoryID` int(11) unsigned NOT NULL auto_increment,
Expand Down
68 changes: 64 additions & 4 deletions SQL/0000-00-05-ElectrophysiologyTables.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -180,13 +180,28 @@ CREATE TABLE `physiological_electrode` (
REFERENCES `physiological_electrode_material` (`PhysiologicalElectrodeMaterialID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;


-- Create `physiological_event_file` table
CREATE TABLE `physiological_event_file` (
`EventFileID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`PhysiologicalFileID` int(10) unsigned NOT NULL,
`FileType` varchar(20) NOT NULL,
`FilePath` varchar(255) DEFAULT NULL,
`LastUpdate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`LastWritten` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`EventFileID`),
KEY `FK_physio_file_ID` (`PhysiologicalFileID`),
KEY `FK_event_file_type` (`FileType`),
CONSTRAINT `FK_event_file_type` FOREIGN KEY (`FileType`) REFERENCES `ImagingFileTypes` (`type`),
CONSTRAINT `FK_physio_file_ID` FOREIGN KEY (`PhysiologicalFileID`) REFERENCES `physiological_file` (`PhysiologicalFileID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

-- Create physiological_task_event table that will store all information
-- regarding the task executed during the physiological recording
CREATE TABLE `physiological_task_event` (
`PhysiologicalTaskEventID` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,
`PhysiologicalFileID` INT(10) UNSIGNED NOT NULL,
`EventFileID` INT(10) unsigned NOT NULL,
`InsertTime` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
`Onset` DECIMAL(11,6) NOT NULL,
`Duration` DECIMAL(11,6) NOT NULL,
Expand All @@ -196,15 +211,60 @@ CREATE TABLE `physiological_task_event` (
`EventType` VARCHAR(50) DEFAULT NULL,
`TrialType` VARCHAR(255) DEFAULT NULL,
`ResponseTime` TIME DEFAULT NULL,
`FilePath` VARCHAR(255) DEFAULT NULL,
`AssembledHED` TEXT DEFAULT NULL,
PRIMARY KEY (`PhysiologicalTaskEventID`),
KEY `FK_event_file` (`EventFileID`),
CONSTRAINT `FK_phys_file_FileID_4`
FOREIGN KEY (`PhysiologicalFileID`)
REFERENCES `physiological_file` (`PhysiologicalFileID`)
ON DELETE CASCADE
ON DELETE CASCADE,
CONSTRAINT `FK_event_file`
FOREIGN KEY (`EventFileID`)
REFERENCES `physiological_event_file` (`EventFileID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;


-- Create `physiological_event_archive` to store event archive info
CREATE TABLE `physiological_event_archive` (
`EventArchiveID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`PhysiologicalFileID` int(10) unsigned NOT NULL,
`Blake2bHash` varchar(128) NOT NULL,
`FilePath` varchar(255) NOT NULL,
PRIMARY KEY (`EventArchiveID`),
KEY `FK_phy_file_ID` (`PhysiologicalFileID`),
CONSTRAINT `FK_phy_file_ID` FOREIGN KEY (`PhysiologicalFileID`) REFERENCES `physiological_file` (`PhysiologicalFileID`) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

-- Create `physiological_event_parameter` to capture all event parameters
-- from events.json
CREATE TABLE `physiological_event_parameter` (
`EventParameterID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`EventFileID` int(10) unsigned NOT NULL,
`ParameterName` varchar(255) NOT NULL,
`Description` text DEFAULT NULL,
`LongName` varchar(255) DEFAULT NULL,
`Units` varchar(50) DEFAULT NULL,
`isCategorical` enum('Y', 'N') DEFAULT NULL,
`HED` text DEFAULT NULL,
PRIMARY KEY (`EventParameterID`),
KEY `FK_event_file_ID` (`EventFileID`),
CONSTRAINT `FK_event_file_ID` FOREIGN KEY (`EventFileID`) REFERENCES `physiological_event_file` (`EventFileID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

-- Create `physiological_event_parameter_category_level` to capture
-- category levels from events.json
CREATE TABLE `physiological_event_parameter_category_level` (
`CategoricalLevelID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`EventParameterID` int(10) unsigned NOT NULL,
`LevelName` varchar(255) NOT NULL,
`Description` text DEFAULT NULL,
`HED` text DEFAULT NULL,
PRIMARY KEY (`CategoricalLevelID`),
KEY `FK_event_param_ID` (`EventParameterID`),
CONSTRAINT `FK_event_param_ID` FOREIGN KEY (`EventParameterID`) REFERENCES `physiological_event_parameter` (`EventParameterID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

-- Create physiological_archive which will store archives of all the files for
-- Front-end download
Expand Down
93 changes: 93 additions & 0 deletions SQL/New_patches/2022-03-03-AddHEDTags.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
-- ############################## CAPTURE HEDVersion ########################## --
-- HEDVersion from dataset_description.json to be added to parameter_type
-- Entry in physiological_parameter_file will be added on dataset import**
INSERT INTO parameter_type (Name, Type, Description, SourceFrom) VALUES
('HEDVersion', 'text', 'HED Schema Version','physiological_parameter_file')
;

-- ############################## HANDLE EVENT FILES ########################## --
-- Create `physiological_event_file` table
CREATE TABLE `physiological_event_file` (
`EventFileID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`PhysiologicalFileID` int(10) unsigned NOT NULL,
`FileType` varchar(20) NOT NULL,
`FilePath` varchar(255) DEFAULT NULL,
`LastUpdate` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
`LastWritten` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (`EventFileID`),
KEY `FK_physio_file_ID` (`PhysiologicalFileID`),
KEY `FK_event_file_type` (`FileType`),
CONSTRAINT `FK_event_file_type` FOREIGN KEY (`FileType`) REFERENCES `ImagingFileTypes` (`type`),
CONSTRAINT `FK_physio_file_ID` FOREIGN KEY (`PhysiologicalFileID`) REFERENCES `physiological_file` (`PhysiologicalFileID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

-- Create reference to EventFileID in `physiological_task_event` table
SET FOREIGN_KEY_CHECKS= 0;
ALTER TABLE physiological_task_event
ADD COLUMN `EventFileID` int(10) unsigned NOT NULL AFTER PhysiologicalFileID,
ADD KEY `FK_event_file` (`EventFileID`),
ADD CONSTRAINT `FK_event_file` FOREIGN KEY (`EventFileID`) REFERENCES `physiological_event_file` (`EventFileID`)
;

-- Create column for assembled HEd tags in `physiological_task_event` table
ALTER TABLE physiological_task_event
ADD COLUMN `AssembledHED` text DEFAULT NULL
;

-- Insert files into `physiological_event_file` table
INSERT INTO physiological_event_file (PhysiologicalFileID, FilePath, FileType)
SELECT DISTINCT PhysiologicalFileID, FilePath, 'tsv' FROM physiological_task_event
;

-- Update EventFileID reference in `physiological_task_event` table
UPDATE physiological_task_event te
SET EventFileID=(SELECT EventFileID FROM physiological_event_file WHERE PhysiologicalFileID=te.PhysiologicalFileID)
;

-- Delete FilePath column in `physiological_task_event` table
ALTER TABLE physiological_task_event
DROP COLUMN FilePath
;


-- ############################## EVENT FILES ARCHIVE ########################## --
CREATE TABLE `physiological_event_archive` (
`EventArchiveID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`PhysiologicalFileID` int(10) unsigned NOT NULL,
`Blake2bHash` varchar(128) NOT NULL,
`FilePath` varchar(255) NOT NULL,
PRIMARY KEY (`EventArchiveID`),
KEY `FK_phy_file_ID` (`PhysiologicalFileID`),
CONSTRAINT `FK_phy_file_ID` FOREIGN KEY (`PhysiologicalFileID`) REFERENCES `physiological_file` (`PhysiologicalFileID`) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;


-- ############################## CAPTURE EVENT PARAMETERS ########################## --
CREATE TABLE `physiological_event_parameter` (
`EventParameterID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`EventFileID` int(10) unsigned NOT NULL,
`ParameterName` varchar(255) NOT NULL,
`Description` text DEFAULT NULL,
`LongName` varchar(255) DEFAULT NULL,
`Units` varchar(50) DEFAULT NULL,
`isCategorical` enum('Y', 'N') DEFAULT NULL,
`HED` text DEFAULT NULL,
PRIMARY KEY (`EventParameterID`),
KEY `FK_event_file_ID` (`EventFileID`),
CONSTRAINT `FK_event_file_ID` FOREIGN KEY (`EventFileID`) REFERENCES `physiological_event_file` (`EventFileID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;

CREATE TABLE `physiological_event_parameter_category_level` (
`CategoricalLevelID` int(10) unsigned NOT NULL AUTO_INCREMENT,
`EventParameterID` int(10) unsigned NOT NULL,
`LevelName` varchar(255) NOT NULL,
`Description` text DEFAULT NULL,
`HED` text DEFAULT NULL,
PRIMARY KEY (`CategoricalLevelID`),
KEY `FK_event_param_ID` (`EventParameterID`),
CONSTRAINT `FK_event_param_ID` FOREIGN KEY (`EventParameterID`) REFERENCES `physiological_event_parameter` (`EventParameterID`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8
;
11 changes: 4 additions & 7 deletions modules/electrophysiology_browser/jsx/electrophysiologySessionView.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,13 +206,10 @@ class ElectrophysiologySessionView extends Component {
),
epochsURL:
dbEntry
&& dbEntry.file.downloads.map(
(group) =>
group.links[3]?.file
&& loris.BaseURL
+ '/electrophysiology_browser/file_reader/?file='
+ group.links[3].file
),
&& dbEntry.file?.epochsURL
&& [loris.BaseURL
+ '/electrophysiology_browser/file_reader/?file='
+ dbEntry.file.epochsURL],
electrodesURL:
dbEntry
&& dbEntry.file.downloads.map(
Expand Down
11 changes: 9 additions & 2 deletions modules/electrophysiology_browser/php/sessions.class.inc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -460,6 +460,13 @@ class Sessions extends \NDB_Page

$fileSummary['downloads'] = $this->getDownloadLinks($physioFileObj);
$fileSummary['chunks_urls'] = $physioFileObj->getChunksURLs();
$fileSummary['epochsURL'] = $db->pselectOne(
"SELECT FilePath
FROM physiological_event_file
WHERE PhysiologicalFileID=:physioFileID
AND FileType='tsv'",
['physioFileID' => $physioFileID]
);

$fileOutput = $db->pselectone(
'SELECT pot.OutputTypeName
Expand Down Expand Up @@ -548,15 +555,15 @@ class Sessions extends \NDB_Page
$queries = [
'physiological_electrode' => 'physiological_electrode_file',
'physiological_channel' => 'physiological_channel_file',
'physiological_task_event' => 'physiological_task_event_file',
'physiological_event_archive' => 'physiological_event_files',
'physiological_annotation_archive' => 'physiological_annotation_files',
'physiological_archive' => 'all_files',
];

$labels = [
'physiological_electrode_file' => 'Electrodes',
'physiological_channel_file' => 'Channels',
'physiological_task_event_file' => 'Events',
'physiological_event_files' => 'Events',
'physiological_annotation_files' => 'Annotations',
'all_files' => 'All Files',
];
Expand Down
42 changes: 42 additions & 0 deletions tools/single_use/archive_eeg_event_files.php
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
<?php declare(strict_types=1);
cmadjar marked this conversation as resolved.
Show resolved Hide resolved
require_once __DIR__ . "/../generic_includes.php";

$db = \NDB_Factory::singleton()->database();

$dataDir = $db->pselectOne(
'SELECT Value
FROM Config AS config
INNER JOIN ConfigSettings AS c
ON c.Name=:name AND config.ConfigID=c.ID',
['name' => 'dataDirBasepath']
);

$filepaths = $db->pselect(
"SELECT
DISTINCT(FilePath), PhysiologicalFileID
FROM physiological_event_file
WHERE FileType='tsv'",
[]
);

foreach ($filepaths as $record) {
$tgz_name = str_replace(".tsv", '.tgz', $record['FilePath']);
$tgz_path = $dataDir . $tgz_name;
print_r($tgz_path . "\n");
$eventPath = $dataDir . $record['FilePath'];
$arch_file = new \PharData($tgz_path);
$arch_file->addFile($eventPath, basename($eventPath));

$f = file_get_contents($tgz_path);
$hash = sodium_crypto_generichash($f);

//Update database with hash
$db->insert(
'physiological_event_archive',
[
'PhysiologicalFileID' => $record['PhysiologicalFileID'],
'FilePath' => str_replace($dataDir, '', $tgz_path),
'Blake2bHash' => bin2hex($hash)
]
);
}