Adding ability for deid to support deid-provided functions (#208)

* adding ability for deid to support deid-provided functions

as of this version, a user can specify a value as a deid_func:<name> meaning that we use
a deid provided function. This is useful for providing some general uid and a customized jitter
function, along with other functions that users might want to add. With this change I have
provided docs and a contributing section for how to do this, along with running pyflakes on
the tests
Signed-off-by: vsoch <[email protected]>

.github/workflows/codespell.yml

@@ -12,8 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v2
-      - uses: codespell-project/actions-codespell@de089481bd65b71b4d02e34ffb3566b6d189333e
-        uses: crate-ci/typos@592b36d23c62cb378f6097a292bc902ee73f93ef # version 1.0.4
+      - uses: actions/checkout@v3
+      - uses: crate-ci/typos@592b36d23c62cb378f6097a292bc902ee73f93ef # version 1.0.4
         with: 
           files: ./deid ./docs/_docs ./docs/README.md ./docs/pages ./examples

.gitignore

@@ -3,6 +3,7 @@ dist
 deid.egg-info
 build
 pypi.sh
+env
 
 private
 .vscode
@@ -34,4 +35,4 @@ Icon
 Network Trash Folder
 Temporary Items
 .apdisk
-.cache
+.cache

CHANGELOG.md

@@ -14,6 +14,7 @@ and **Merged pull requests**. Critical items to know are:
 Referenced versions in headers are tagged on Github, in parentheses are for pypi.
 
 ## [vxx](https://github.com/pydicom/deid/tree/master) (master)
+ - adding support for deid provided functions [#207](https://github.com/pydicom/deid/issues/207) (0.2.3)
  - update CTP deid.dicom up until [this commit](https://github.com/johnperry/CTP/commit/345b05b157c046532e8791a63ababbf6d0dba59b) (0.2.29)
  - various LGTM alert fixes [#186](https://github.com/pydicom/deid/pull/186) (0.0.28)
  - bug fix for exception when attempting to jitter DA/DT which cannot be jittered (space) [#189] (https://github.com/pydicom/deid/issues/189) (0.2.27)

deid/config/__init__.py

@@ -35,8 +35,9 @@
 
 
 class DeidRecipe:
-    """Create and work with a deid recipe to filter and perform operations on
-    a dicom header. Usage typically looks like:
+    """Create a deid recipe to filter and perform operations on a dicom header.
+
+    Usage typically looks like:
 
     deid = 'dicom.deid'
     recipe = DeidRecipe(deid)
@@ -68,8 +69,10 @@ def __repr__(self):
         return "[deid]"
 
     def load(self, deid):
-        """load a deid recipe into the object. If a deid configuration is
-        already defined, append to that.
+        """
+        Load a deid recipe into the object.
+
+        If a deid configuration is already defined, append to that.
         """
         deid = get_deid(deid)
         if deid is not None:
@@ -82,7 +85,9 @@ def load(self, deid):
             self.deid = load_combined_deid([self.deid, deid])
 
     def _get_section(self, name):
-        """return a section (key) in the loaded deid, if it exists"""
+        """
+        Return a section (key) in the loaded deid, if it exists
+        """
         section = None
         if self.deid is not None:
             section = self.deid.get(name)
@@ -91,11 +96,15 @@ def _get_section(self, name):
     # Get Sections
 
     def get_format(self):
-        """return the format of the loaded deid, if one exists"""
+        """
+        Return the format of the loaded deid, if one exists
+        """
         return self._get_section("format")
 
     def _get_named_section(self, section_name, name=None):
-        """a helper function to return an entire section, or if a name is
+        """Get a named section from the deid recipe.
+
+        a helper function to return an entire section, or if a name is
         provided, a named section under it. If the section is not
         defined, we appropriately return None.
         """
@@ -105,19 +114,27 @@ def _get_named_section(self, section_name, name=None):
         return section
 
     def get_filters(self, name=None):
-        """return all filters for a deid recipe, or a set based on a name"""
+        """
+        Return all filters for a deid recipe, or a set based on a name
+        """
         return self._get_named_section("filter", name)
 
     def get_values_lists(self, name=None):
-        """return a values list by name"""
+        """
+        Return a values list by name
+        """
         return self._get_named_section("values", name)
 
     def get_fields_lists(self, name=None):
-        """return a values list by name"""
+        """
+        Return a values list by name
+        """
         return self._get_named_section("fields", name)
 
     def _get_actions(self, action=None, field=None, section="header"):
-        """handler for header or filemeta actions."""
+        """
+        Handler for header or filemeta actions.
+        """
         header = self._get_section(section) or []
         if header is not None:
             if action is not None:
@@ -129,7 +146,7 @@ def _get_actions(self, action=None, field=None, section="header"):
         return header
 
     def get_actions(self, action=None, field=None):
-        """get deid actions to perform on a header, or a subset based on a type
+        """Get deid actions to perform on a header, or a subset based on a type
 
         A header action is a list with the following:
         {'action': 'REMOVE', 'field': 'AssignedLocation'},
@@ -159,7 +176,9 @@ def has_actions(self):
     # Listing
 
     def listof(self, section):
-        """return a list of keys for a section"""
+        """
+        Return a list of keys for a section
+        """
         listing = self._get_section(section) or {}
         return list(listing.keys())
 
@@ -175,7 +194,9 @@ def ls_fieldlists(self):
     # Init
 
     def _init_deid(self, deid=None, base=False, default_base="dicom"):
-        """initialize the recipe with one or more deids, optionally including
+        """Initialize a recipe.
+
+        initialize the recipe with one or more deids, optionally including
         the default. This function is called at init time. If you need to add
         or work with already loaded configurations, use add/remove
 

deid/dicom/actions/__init__.py

@@ -0,0 +1,13 @@
+from .jitter import jitter_timestamp, jitter_timestamp_func
+from .uids import basic_uuid, dicom_uuid, suffix_uuid, pydicom_uuid
+
+# Function lookup
+# Functions here must take an item, field, and value
+
+deid_funcs = {
+    "jitter": jitter_timestamp_func,
+    "dicom_uuid": dicom_uuid,
+    "suffix_uuid": suffix_uuid,
+    "basic_uuid": basic_uuid,
+    "pydicom_uuid": pydicom_uuid,
+}

deid/dicom/actions.py → deid/dicom/actions/jitter.py

@@ -24,20 +24,37 @@
 
 from deid.logger import bot
 from deid.utils import get_timestamp
+from deid.utils import parse_keyvalue_pairs
 
 # Timestamps
 
 
+def jitter_timestamp_func(item, value, field, **kwargs):
+    """
+    A wrapper to jitter_timestamp so it works as a custom function.
+    """
+    opts = parse_keyvalue_pairs(kwargs.get("extras"))
+
+    # Default to jitter by one day
+    value = int(opts.get("days", 1))
+
+    # The user can optionally provide years
+    if "years" in opts:
+        value = (int(opts["years"]) * 365) + value
+    return jitter_timestamp(field, value)
+
+
 def jitter_timestamp(field, value):
-    """if present, jitter a timestamp in dicom
-    field "field" by number of days specified by "value"
-    The value can be positive or negative.
+    """Jitter a timestamp "field" by number of days specified by "value"
+
+    The value can be positive or negative. This function is grandfathered
+    into deid custom funcs, as it existed before they did. Since a custom
+    func requires an item, we have a wrapper above to support this use case.
 
     Parameters
     ==========
     field: the field with the timestamp
     value: number of days to jitter by. Jitter bug!
-
     """
     if not isinstance(value, int):
         value = int(value)

deid/dicom/actions/uids.py

@@ -0,0 +1,90 @@
+"""
+
+Copyright (c) 2022 Vanessa Sochat
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+"""
+
+from deid.utils import parse_keyvalue_pairs
+from pydicom.uid import generate_uid as pydicom_generate_uid
+from deid.logger import bot
+import uuid
+
+
+def basic_uuid(item, value, field, **kwargs):
+    """A basic function to replace a field with a uuid.uuid4() string"""
+    return str(uuid.uuid4())
+
+
+def pydicom_uuid(item, value, field, **kwargs):
+    """
+    Use pydicom to generate the UID. Optional kwargs include:
+
+    prefix (str): provide a custom prefix
+    stable_remapping (bool): if true, use the orignal value for entropy.
+    This ensures stability across different runs that use the same UID.
+
+    The prefix must match '^(0|[1-9][0-9]*)(\\.(0|[1-9][0-9]*))*\\.$'
+    """
+    opts = parse_keyvalue_pairs(kwargs.get("extras"))
+
+    # We always provide a prefix so the stable remapping is done
+    prefix = opts.get("prefix", "2.25.")
+    stable_remapping = opts.get("stable_remapping", True)
+    entropy_srcs = []
+
+    # They would need to unset the default prefix
+    if stable_remapping is True and not prefix:
+        bot.warning("A prefix must be provided to use stable remapping.")
+
+    if stable_remapping is True:
+        original = str(field.element.value)
+        entropy_srcs.append(original)
+    return pydicom_generate_uid(prefix=prefix, entropy_srcs=entropy_srcs)
+
+
+def suffix_uuid(item, value, field, **kwargs):
+    """Return the same field, with a uuid suffix.
+
+    Provided in docs: https://pydicom.github.io/deid/examples/func-replace/
+    """
+    # a field can either be just the name string, or a DicomElement
+    if hasattr(field, "name"):
+        field = field.name
+    prefix = field.lower().replace(" ", " ")
+    return prefix + "-" + str(uuid.uuid4())
+
+
+def dicom_uuid(item, value, field, dicom, **kwargs):
+    """
+    Generate a dicom uid that better conforms to the dicom standard.
+    """
+    # a field can either be just the name string, or a DicomElement
+    if hasattr(field, "name"):
+        field = field.name
+
+    opts = parse_keyvalue_pairs(kwargs.get("extras"))
+    org_root = opts.get("org_root", "anonymous-organization")
+
+    bigint_uid = str(uuid.uuid4().int)
+    full_uid = org_root + "." + bigint_uid
+
+    # A DICOM UID is limited to 64 characters
+    return full_uid[0:64]