Merge pull request #193 from joshmoore/ometa-squashed

ometa (squashed history)

Author: joshmoore

omero/annotation_scripts/KeyVal_from_csv.py

@@ -0,0 +1,244 @@
+# coding=utf-8
+"""
+ MIF/Add_Key_Val_from_csv.py
+
+ Adds key-value (kv) metadata to images in a dataset from a csv file
+ The first column contains the filenames
+ The first  row of the file contains the keys
+ The rest is the values for each file/key
+
+-----------------------------------------------------------------------------
+  Copyright (C) 2018
+  This program is free software; you can redistribute it and/or modify
+  it under the terms of the GNU General Public License as published by
+  the Free Software Foundation; either version 2 of the License, or
+  (at your option) any later version.
+  This program is distributed in the hope that it will be useful,
+  but WITHOUT ANY WARRANTY; without even the implied warranty of
+  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+  GNU General Public License for more details.
+  You should have received a copy of the GNU General Public License along
+  with this program; if not, write to the Free Software Foundation, Inc.,
+  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+------------------------------------------------------------------------------
+Created by Christian Evenhuis
+
+"""
+
+import omero
+from omero.gateway import BlitzGateway
+from omero.rtypes import rstring, rlong
+import omero.scripts as scripts
+from omero.model import PlateI, ScreenI, DatasetI
+from omero.rtypes import *
+from omero.cmd import Delete2
+
+import sys
+import csv
+import copy
+
+# this is for downloading a temp file
+from omero.util.temp_files import create_path
+
+from omero.util.populate_roi import DownloadingOriginalFileProvider
+from omero.util.populate_metadata import ParsingContext
+
+from collections import OrderedDict
+
+
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+def get_existing_map_annotations( obj ):
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+    print("getting the existing kv's")
+    ord_dict = OrderedDict()
+    for ann in obj.listAnnotations():
+        if( isinstance(ann, omero.gateway.MapAnnotationWrapper) ):
+            kvs = ann.getValue()
+            for k,v in kvs:
+                if k not in ord_dict:
+                    ord_dict[k]=set()
+                ord_dict[k].add(v)
+    return ord_dict
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+def remove_map_annotations(conn, dtype, Id ):
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+    image = conn.getObject(dtype,int(Id))
+    namespace = omero.constants.metadata.NSCLIENTMAPANNOTATION
+
+    filename = image.getName()
+
+    anns = list( image.listAnnotations())
+    mapann_ids = [ann.id for ann in anns
+         if isinstance(ann, omero.gateway.MapAnnotationWrapper) ]
+
+    try:
+        delete = Delete2(targetObjects={'MapAnnotation': mapann_ids})
+        handle = conn.c.sf.submit(delete)
+        conn.c.waitOnCmd(handle, loops=10, ms=500, failonerror=True,
+                     failontimeout=False, closehandle=False)
+
+    except Exception as ex:
+        print("Failed to delete links: {}".format(ex.message))
+    return
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+def get_original_file(conn, object_type, object_id, file_ann_id=None):
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+    omero_object = conn.getObject("Dataset", int(object_id))
+    if omero_object is None:
+        sys.stderr.write("Error: Dataset does not exist.\n")
+        sys.exit(1)
+    file_ann = None
+
+    for ann in omero_object.listAnnotations():
+        if isinstance(ann, omero.gateway.FileAnnotationWrapper):
+            file_name = ann.getFile().getName()
+            # Pick file by Ann ID (or name if ID is None)
+            if (file_ann_id is None and file_name.endswith(".csv")) or (
+                    ann.getId() == file_ann_id):
+                file_ann = ann
+    if file_ann is None:
+        sys.stderr.write("Error: File does not exist.\n")
+        sys.exit(1)
+
+    return file_ann.getFile()._obj
+
+
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+def populate_metadata(client, conn, script_params):
+# - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+    dataType = script_params["Data_Type"]
+    ids      = script_params["IDs"]
+
+    datasets = list(conn.getObjects(dataType, ids))
+    for ds in datasets:
+        ID = ds.getId()
+
+        # not sure what this is doing
+        file_ann_id = None
+        if "File_Annotation" in script_params:
+            file_ann_id = long(script_params["File_Annotation"])
+            print("set ann id")
+
+        original_file = get_original_file(
+            conn, dataType, ID, file_ann_id)
+        provider = DownloadingOriginalFileProvider(conn)
+
+        # read the csv
+        file_handle = provider.get_original_file_data(original_file)
+        data =list(csv.reader(file_handle,delimiter=','))
+        file_handle.close()
+
+        # create a dictionary for image_name:id
+        dict_name_id={}
+        for img in ds.listChildren():
+            img_name = img.getName()
+            if( img_name in dict_name_id ):
+                sys.stderr.write("File names not unique: {}".format(img_name))
+                sys.exit(1)
+            dict_name_id[img_name] = int(img.getId())
+
+        # keys are in the header row
+        header =data[0]
+        kv_data = header[1:]  # first header is the fimename columns
+        rows    = data[1:]
+
+        nimg_updated = 0
+        for row in rows: # loop over images
+            img_name = row[0]
+            if( img_name not in dict_name_id ):
+                print("Can't find filename : {}".format(img_name) )
+            else:
+                img_ID = dict_name_id[img_name]         # look up the ID
+                img    = conn.getObject('Image',img_ID) # get the img
+
+                existing_kv = get_existing_map_annotations( img )
+                updated_kv  = copy.deepcopy(existing_kv)
+                print("Existing kv ")
+                for k,vset in existing_kv.items():
+                    print(type(vset),len(vset))
+                    for v in vset:
+                        print(k,v)
+
+                for i in range(1,len(row)):  # first entry is the filename
+                    key = header[i].strip()
+                    vals = row[i].strip().split(';')
+                    if( len(vals) > 0 ):
+                        for val in vals:
+                            if len(val)>0 : 
+                                if key not in updated_kv: updated_kv[key] = set()
+                                print("adding",key,val)
+                                updated_kv[key].add(val)
+
+                if( existing_kv != updated_kv ):
+                    nimg_updated = nimg_updated + 1
+                    print("The key-values pairs are different")
+                    remove_map_annotations( conn, 'Image', img.getId()  )
+                    map_ann = omero.gateway.MapAnnotationWrapper(conn)
+                    namespace = omero.constants.metadata.NSCLIENTMAPANNOTATION
+                    map_ann.setNs(namespace)
+                    # convert the ordered dict to a list of lists
+                    kv_list=[]
+                    for k,vset in updated_kv.items():
+                        for v in vset:
+                            kv_list.append( [k,v] )
+                    map_ann.setValue(kv_list)
+                    map_ann.save()
+                    img.linkAnnotation(map_ann)
+                else:
+                    print("No change change in kv's")
+
+    return "Added {} kv pairs to {}/{} files  ".format(len(header)-1,nimg_updated,len(dict_name_id))
+
+
+def run_script():
+
+    data_types = [rstring('Dataset')]
+    client = scripts.client(
+        'Add_Key_Val_from_csv',
+        """
+    This script processes a csv file, attached to a Dataset
+        """,
+        scripts.String(
+            "Data_Type", optional=False, grouping="1",
+            description="Choose source of images",
+            values=data_types, default="Dataset"),
+
+        scripts.List(
+            "IDs", optional=False, grouping="2",
+            description="Plate or Screen ID.").ofType(rlong(0)),
+
+        scripts.String(
+            "File_Annotation", grouping="3",
+            description="File ID containing metadata to populate."),
+
+        authors=["Christian Evenhuis"],
+        institutions=["MIF UTS"],
+        contact="https://forum.image.sc/tag/omero"
+    )
+
+    try:
+        # process the list of args above.
+        script_params = {}
+        for key in client.getInputKeys():
+            if client.getInput(key):
+                script_params[key] = client.getInput(key, unwrap=True)
+
+        # wrap client to use the Blitz Gateway
+        conn = BlitzGateway(client_obj=client)
+        print("script params")
+        for k,v in script_params.items():
+            print(k,v)
+        message = populate_metadata(client, conn, script_params)
+        client.setOutput("Message", rstring(message))
+    except:
+        pass
+
+    finally:
+        client.closeSession()
+
+
+if __name__ == "__main__":
+    run_script()