added adjusted files for further work on OAMI2

Author: mzajcev

.gitignore

@@ -1 +1,3 @@
+env
 *.pyc
+userconfig

LICENSE.md

@@ -0,0 +1,25 @@
+ This program is free software: you can redistribute it and/or modify
+    it under the terms of the GNU General Public License as published by
+    the Free Software Foundation, either version 3 of the License, or
+    (at your option) any later version.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+    Dieses Programm ist Freie Software: Sie können es unter den Bedingungen
+    der GNU General Public License, wie von der Free Software Foundation,
+    Version 3 der Lizenz oder (nach Ihrer Wahl) jeder neueren
+    veröffentlichten Version, weiter verteilen und/oder modifizieren.
+
+    Dieses Programm wird in der Hoffnung bereitgestellt, dass es nützlich sein wird, jedoch
+    OHNE JEDE GEWÄHR,; sogar ohne die implizite
+    Gewähr der MARKTFÄHIGKEIT oder EIGNUNG FÜR EINEN BESTIMMTEN ZWECK.
+    Siehe die GNU General Public License für weitere Einzelheiten.
+
+    Sie sollten eine Kopie der GNU General Public License zusammen mit diesem
+    Programm erhalten haben. Wenn nicht, siehe <https://www.gnu.org/licenses/>.

OAMI/.gitignore

@@ -0,0 +1 @@
+*.pyc

DOI_PREFIX_WHITELIST OAMI/DOI_PREFIX_WHITELIST

@@ -0,0 +1,13 @@
+from autovividict import autovividict
+from urllib2 import quote
+
+def make_datestring(year, month, day):
+    datestring = "%04d" % year  # YYYY
+    if month is not None:
+        datestring += "-%02d" % month  # YYYY-MM
+    if day is not None:
+        datestring += "-%02d" % day  # YYYY-MM-DD
+    return datestring
+
+def filename_from_url(url):
+    return quote(url, safe='')

README OAMI/README

@@ -0,0 +1,17 @@
+from collections import defaultdict
+
+# counting dictionary that serves as 0 for addition
+# this allows incrementing unknown keys: d['k'] += 1
+class countdict(defaultdict):
+    def __init__(self, *args, **kwargs):
+        self.value = 0
+        super(countdict, self).__init__(*args, **kwargs)
+    def __repr__(self):
+        return str(dict(self))
+    def __add__(self, x):
+        return self.value + x
+
+# autovivicatious counting dictionary, allowing dynamic creation of keys
+# explained at <http://en.wikipedia.org/wiki/Autovivification#Python>
+def autovividict():
+    return countdict(autovividict)

doc/man/oa-get.1.rst OAMI/doc/man/oa-get.1.rst

@@ -0,0 +1,85 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from os import makedirs, path
+from sys import stderr, exit
+from xdg import BaseDirectory
+from ConfigParser import RawConfigParser, NoSectionError, NoOptionError
+
+APPLICATION_NAME="open-access-media-importer"
+cache_path = path.join(BaseDirectory.xdg_cache_home, APPLICATION_NAME)
+config_path = path.join(BaseDirectory.xdg_config_home, APPLICATION_NAME)
+data_path = path.join(BaseDirectory.xdg_data_home, APPLICATION_NAME)
+
+def database_path(source):
+    return path.join(data_path, '%s.sqlite' % source)
+
+def ensure_directory_exists(directory):
+    if not path.exists(directory):
+        makedirs(directory)
+
+for p in (cache_path, config_path, data_path):
+    ensure_directory_exists(p)
+
+_metadata_path = path.join(cache_path, 'metadata')
+
+_metadata_raw_path = path.join(_metadata_path, 'raw')
+def get_metadata_raw_source_path(source_name):
+    p = path.join(_metadata_raw_path, source_name)
+    ensure_directory_exists(p)
+    return p
+
+_metadata_refined_path = path.join(_metadata_path, 'refined')
+def get_metadata_refined_source_path(source_name):
+    p = path.join(_metadata_refined_path, source_name)
+    ensure_directory_exists(p)
+    return p
+
+_media_path = path.join(cache_path, 'media')
+
+_media_raw_path = path.join(_media_path, 'raw')
+def get_media_raw_source_path(source_name):
+    p = path.join(_media_raw_path, source_name)
+    ensure_directory_exists(p)
+    return p
+
+_media_refined_path = path.join(_media_path, 'refined')
+def get_media_refined_source_path(source_name):
+    p = path.join(_media_refined_path, source_name)
+    ensure_directory_exists(p)
+    return p
+
+free_license_urls = [
+    'http://creativecommons.org/licenses/by/2.0/',
+    'http://creativecommons.org/licenses/by-sa/2.0/',
+    'http://creativecommons.org/licenses/by/2.5/',
+    'http://creativecommons.org/licenses/by-sa/2.5/',
+    'http://creativecommons.org/licenses/by/3.0/',
+    'http://creativecommons.org/licenses/by-sa/3.0/',
+    'http://creativecommons.org/licenses/by/4.0/',
+    'http://creativecommons.org/licenses/by-sa/4.0/',
+    'http://creativecommons.org/publicdomain/zero/1.0/'
+]
+
+USERCONFIG_FILENAME = "userconfig"
+userconfig_file = path.join(config_path, USERCONFIG_FILENAME)
+userconfig = RawConfigParser()
+userconfig.optionsxform = str  # case sensitivity
+userconfig.read(userconfig_file)
+
+def get_userconfig(section, option):
+    try:
+        return userconfig.get(section, option)
+    except NoSectionError:
+        stderr.write("“%s” does not contain a “%s” section.\n" % \
+                         (userconfig_file, section))
+        exit(127)
+    except NoOptionError:
+        stderr.write("“%s” does not contain a “%s” option in the “%s” section.\n" % \
+                         (userconfig_file, option, section))
+        exit(127)
+
+api_url = get_userconfig('wiki', 'api_url')
+username = get_userconfig('wiki', 'username')
+password = get_userconfig('wiki', 'password')
+whitelist_doi = get_userconfig('whitelist', 'doi').split()

doi_pref.tsv OAMI/doi_pref.tsv

@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+from urllib2 import urlopen, urlparse, Request, HTTPError
+from xml.etree.cElementTree import dump, ElementTree
+from sys import stderr
+
+def _get_file_from_url(url):
+    req = Request(url, None, {'User-Agent' : 'oa-put/2012-08-15'})
+    try:
+        remote_file = urlopen(req)
+        return remote_file
+    except HTTPError as e:
+        stderr.write('When trying to download <%s>, the following error occured: “%s”.\n' % \
+            (url, str(e)))
+        exit(255)
+
+def get_pmcid_from_doi(doi):
+    if not type(doi) == unicode:
+        raise TypeError, "Cannot get PMCID for DOI %s of type %s." % (doi, type(doi))
+    url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pmc&term=%s' % doi
+    xml_file = _get_file_from_url(url)
+    tree = ElementTree()
+    tree.parse(xml_file)
+    try:
+        return int(tree.find('IdList/Id').text)
+    except AttributeError:
+        return None
+
+def get_pmid_from_doi(doi):
+    if not type(doi) == unicode:
+        raise TypeError, "Cannot get PMID for DOI %s of type %s." % (doi, type(doi))
+    url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term=%s' % doi
+    xml_file = _get_file_from_url(url)
+    tree = ElementTree()
+    tree.parse(xml_file)
+    try:
+        return int(tree.find('IdList/Id').text)
+    except AttributeError:
+        return None
+
+def get_categories_from_pmid(pmid):
+    """
+    Gets MeSH headings, returns those not deemed too broad.
+    """
+    if not type(pmid) == int:
+        raise TypeError, "Cannot get Categories for PMID %s of type %s." % (pmid, type(pmid))
+    url = 'http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=%s&retmode=xml' % pmid
+    xml_file = _get_file_from_url(url)
+    tree = ElementTree()
+    tree.parse(xml_file)
+    categories = []
+    for heading in tree.iterfind('PubmedArticle/MedlineCitation/MeshHeadingList/MeshHeading'):
+        htree = ElementTree(heading)
+        descriptor_text = htree.find('DescriptorName').text
+        if (htree.find('QualifierName') is not None) or \
+            (' ' in descriptor_text and not 'and' in descriptor_text):
+            categories.append(descriptor_text)
+    return categories

OAMI/helpers/__init__.py

@@ -0,0 +1,138 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import gobject, pygst
+pygst.require("0.10")
+
+import gst
+import progressbar
+
+from sys import exit, stderr
+
+class Media():
+    def __init__(self, filename):
+        self.filename = filename
+        self.has_audio = False
+        self.has_video = False
+        self.position = 0
+        self.lastposition = 0
+
+    def find_streams(self):
+        """
+        Determines if media file has audio and / or video streams.
+        """
+        loop = gobject.MainLoop()
+        pipeline = gst.parse_launch("filesrc name=source ! decodebin2 ! fakesink")
+
+        source = pipeline.get_by_name("source")
+        source.set_property("location", self.filename)
+
+        bus = pipeline.get_bus()
+        def on_message(bus, message):
+            t = message.type
+            if t == gst.MESSAGE_TAG:
+                pipeline.set_state(gst.STATE_PAUSED)
+                keys = message.structure.keys()
+                if 'audio-codec' in keys:
+                    self.has_audio = True
+                if 'video-codec' in keys:
+                    self.has_video = True
+            if t == gst.MESSAGE_ASYNC_DONE:
+                pipeline.set_state(gst.STATE_NULL)
+                loop.quit()
+            elif t == gst.MESSAGE_ERROR:  # error
+                err, debug = message.parse_error()
+                pipeline.set_state(gst.STATE_NULL)
+                loop.quit()
+                if str(err) in [
+                    'Your GStreamer installation is missing a plug-in.',
+                    'Could not demultiplex stream.'
+                ]:
+                    raise RuntimeError, str(err)
+                else:
+                    stderr.write('ERROR: %s\n' %str(err))
+
+        bus.add_signal_watch()
+        bus.connect('message', on_message)
+
+        pipeline.set_state(gst.STATE_PLAYING)
+        pipeline.get_state()
+
+        loop.run()
+
+    def convert(self, outfile):
+        """
+        Converts media file to Ogg Theora or Ogg Theora+Vorbis.
+        """
+        loop = gobject.MainLoop()
+
+        if self.has_video and self.has_audio:
+            pipeline = gst.parse_launch("""
+                filesrc name=source ! decodebin2 name=decoder
+                decoder. ! queue ! theoraenc ! queue ! oggmux name=muxer
+                decoder. ! queue ! audioconvert ! audioresample ! vorbisenc ! progressreport name=report ! muxer.
+                muxer. ! filesink name=sink
+            """)
+        elif self.has_video and not self.has_audio:
+            pipeline = gst.parse_launch("""
+                filesrc name=source ! decodebin2 name=decoder
+                decoder. ! queue ! ffmpegcolorspace ! theoraenc ! progressreport name=report ! oggmux name=muxer
+                muxer. ! filesink name=sink
+            """)
+        elif not self.has_video and self.has_audio:
+            pipeline = gst.parse_launch("""
+                filesrc name=source ! decodebin2 name=decoder
+                decoder. ! queue ! audioconvert ! audioresample ! vorbisenc ! progressreport name=report ! oggmux name=muxer
+                muxer. ! filesink name=sink
+            """)
+        else:
+            raise RuntimeError, 'Unknown audio/video stream combination.'
+
+        source = pipeline.get_by_name('source')
+        source.set_property('location', self.filename)
+
+        sink = pipeline.get_by_name('sink')
+        sink.set_property('location', outfile)
+
+        report = pipeline.get_by_name('report')
+        report.set_property('silent', True)
+
+        bus = pipeline.get_bus()
+        def on_message(bus, message):
+            t = message.type
+            if t == gst.MESSAGE_EOS:  # end of stream
+                pipeline.set_state(gst.STATE_NULL)
+                loop.quit()
+            elif t == gst.MESSAGE_ERROR:  # error
+                err, debug = message.parse_error()
+                stderr.write('ERROR: %s\n' %str(err))
+                pipeline.set_state(gst.STATE_NULL)
+                loop.quit()
+
+        bus.add_signal_watch()
+        bus.connect("message", on_message)
+
+        pipeline.set_state(gst.STATE_PLAYING)
+        pipeline.get_state()
+
+        try:
+            duration = pipeline.query_duration(gst.FORMAT_TIME, None)[0]
+            progress = progressbar.ProgressBar(maxval=duration).start()
+        except gst.QueryError:
+            pass
+
+        def update_progress():
+            try:
+                self.position = pipeline.query_position(gst.FORMAT_TIME, \
+                    None)[0]
+            except:
+                return False  # stop loop
+            try:
+                progress.update(self.position)
+            except:
+                # progressbar fails on >100% progress
+                # fall back to pipeline reporting
+                report.set_property('silent', False)
+            return True  # continue loop
+
+        gobject.timeout_add(100, update_progress)
+        loop.run()