Skip to content

Commit

Permalink
encoder: invenio.config deprecation
Browse files Browse the repository at this point in the history 
* Replaces usage of invenio.config with invenio.base.globals:cfg.
  (addresses inveniosoftware#3106)

Signed-off-by: Leonardo Rossi <[email protected]>
  • Loading branch information
Leonardo Rossi committed Jul 23, 2015
1 parent d4e4a6a commit d2f5863
Show file tree
Hide file tree
Showing 14 changed files with 195 additions and 206 deletions.
89 changes: 48 additions & 41 deletions invenio/modules/encoder/daemon.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
#
# This file is part of Invenio.
# Copyright (C) 2011 CERN.
# Copyright (C) 2011, 2015 CERN.
#
# Invenio is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License as
Expand All @@ -17,43 +17,44 @@
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""Bibencode daemon submodule"""
"""Bibencode daemon submodule."""

import os

import re

import shutil
from invenio.utils.json import json_decode_file

from invenio.base.globals import cfg
from invenio.legacy.bibsched.bibtask import task_get_task_param, \
task_low_level_submission, task_update_progress, write_message
from invenio.modules.encoder.utils import generate_timestamp, getval
from invenio.legacy.bibsched.bibtask import (
task_low_level_submission,
task_get_task_param,
write_message,
task_update_progress
)
from invenio.modules.encoder.config import (
CFG_BIBENCODE_DAEMON_DIR_NEWJOBS,
CFG_BIBENCODE_DAEMON_DIR_OLDJOBS
)
from invenio.utils.json import json_decode_file


# Globals used to generate a unique task name
_TASKID = None
_TIMESTAMP = generate_timestamp()
_NUMBER = 0


def has_signature(string_to_check):
""" Checks if the given string has the signature of a job file
"""
"""Check if the given string has the signature of a job file."""
sig_re = re.compile("^.*\.job$")
if sig_re.match(string_to_check):
return True
else:
return False


def job_to_args(job):
""" Maps the key-value pairs of the job file to CLI arguments for a
"""Map the key-value pairs.
Map the key-value pairs of the job file to CLI arguments for a
low-level task submission
@param job: job dictionary to process
@type job: dictionary
:param job: job dictionary to process
:type job: dictionary
"""
argument_mapping = {
'profile': '-p',
Expand Down Expand Up @@ -83,40 +84,44 @@ def job_to_args(job):
## Set a unique name for the task, this way there can be more than
## one bibencode task running at the same time
task_unique_name = '%(mode)s-%(tid)d-%(ts)s-%(num)d' % {
'mode': job['mode'],
'tid': _TASKID,
'ts': _TIMESTAMP,
'num': _NUMBER
}
'mode': job['mode'],
'tid': _TASKID,
'ts': _TIMESTAMP,
'num': _NUMBER
}
args.append('-N')
args.append(task_unique_name)
## Transform the pairs of the job dictionary to CLI arguments
for key in job:
if key in argument_mapping:
args.append(argument_mapping[key]) # This is the new key
args.append(job[key]) # This is the value from the job file
args.append(argument_mapping[key]) # This is the new key
args.append(job[key]) # This is the value from the job file
return args


def launch_task(args):
""" Launches the job as a new bibtask through the low-level submission
"""Launch the job.
Launche the job as a new bibtask through the low-level submission
interface
"""
return task_low_level_submission('bibencode', 'bibencode:daemon', *args)


def process_batch(jobfile_path):
""" Processes the job if it is a batch job
@param jobfile_path: fullpath to the batchjob file
@type jobfile_path: string
@return: True if the task was successfully launche, False if not
@rtype: bool
:param jobfile_path: fullpath to the batchjob file
:type jobfile_path: string
:return: True if the task was successfully launche, False if not
:rtype: bool
"""
args = []
task_unique_name = '%(mode)s-%(tid)d-%(ts)s-%(num)d' % {
'mode': 'batch',
'tid': _TASKID,
'ts': _TIMESTAMP,
'num': _NUMBER
}
'mode': 'batch',
'tid': _TASKID,
'ts': _TIMESTAMP,
'num': _NUMBER
}
args.append('-N')
args.append(task_unique_name)
args.append('-m')
Expand All @@ -125,14 +130,16 @@ def process_batch(jobfile_path):
args.append(jobfile_path)
return launch_task(args)

def watch_directory(new_job_dir=CFG_BIBENCODE_DAEMON_DIR_NEWJOBS,
old_job_dir=CFG_BIBENCODE_DAEMON_DIR_OLDJOBS):

def watch_directory(new_job_dir=None, old_job_dir=None):
""" Checks a folder job files, parses and executes them
@param new_job_dir: path to the directory with new jobs
@type new_job_dir: string
@param old_job_dir: path to the directory where the old jobs are moved
@type old_job_dir: string
:param new_job_dir: path to the directory with new jobs
:type new_job_dir: string
:param old_job_dir: path to the directory where the old jobs are moved
:type old_job_dir: string
"""
new_job_dir = new_job_dir or cfg['CFG_BIBENCODE_DAEMON_DIR_NEWJOBS']
old_job_dir = old_job_dir or cfg['CFG_BIBENCODE_DAEMON_DIR_OLDJOBS']
global _NUMBER, _TASKID
write_message('Checking directory %s for new jobs' % new_job_dir)
task_update_progress('Checking for new jobs')
Expand Down
61 changes: 26 additions & 35 deletions invenio/modules/encoder/encode.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,36 +19,25 @@

"""BibEncode encoding submodule"""

from six import iteritems

from invenio.base.globals import cfg
from invenio.legacy.bibsched.bibtask import (
write_message,
task_update_progress,
)
from invenio.modules.encoder.config import (
CFG_BIBENCODE_FFMPEG_ENCODING_LOG,
CFG_BIBENCODE_FFMPEG_PASSLOGFILE_PREFIX,
CFG_BIBENCODE_FFMPEG_METADATA_ARGUMENT,
CFG_BIBENCODE_FFMPEG_ENCODE_TIME
)
from invenio.modules.encoder.utils import (
timecode_to_seconds,
generate_timestamp,
chose,
getval,
aspect_string_to_float
)
from invenio.modules.encoder.profiles import get_encoding_profile
from invenio.modules.encoder.metadata import (
ffprobe_metadata,
mediainfo_metadata
)
import time
import os

import subprocess

import time

import uuid

from invenio.base.globals import cfg
from invenio.legacy.bibsched.bibtask import task_update_progress, write_message
from invenio.modules.encoder.metadata import ffprobe_metadata, \
mediainfo_metadata
from invenio.modules.encoder.profiles import get_encoding_profile
from invenio.modules.encoder.utils import aspect_string_to_float, chose, \
generate_timestamp, getval, timecode_to_seconds

from six import iteritems


def _filename_log(output_filename, nofpass=1):
""" Constructs the filename including path for the encoding err file
@param output_filename: name of the video file to be created
Expand All @@ -60,8 +49,8 @@ def _filename_log(output_filename, nofpass=1):
"""
fname = os.path.split(output_filename)[1]
fname = os.path.splitext(fname)[0]
return CFG_BIBENCODE_FFMPEG_ENCODING_LOG % (generate_timestamp() +
"_" + fname + "_%d" % nofpass)
return cfg['CFG_BIBENCODE_FFMPEG_ENCODING_LOG'] % \
(generate_timestamp() + "_" + fname + "_%d" % nofpass)

def determine_aspect(input_file):
""" Checks video metadata to find the display aspect ratio.
Expand Down Expand Up @@ -408,8 +397,8 @@ def insert(key, value):
for key, value in iteritems(metadata):
if value is not None:
meta_arg = (
CFG_BIBENCODE_FFMPEG_METADATA_ARGUMENT % (key, value)
)
cfg['CFG_BIBENCODE_FFMPEG_METADATA_ARGUMENT'] % \
(key, value))
insert("-metadata", meta_arg)
## Special argument additions
if passes == 1:
Expand Down Expand Up @@ -469,9 +458,10 @@ def graphical(value):

## try to parse the status
for line in reversed(lines):
if CFG_BIBENCODE_FFMPEG_ENCODE_TIME.match(line):
if cfg['CFG_BIBENCODE_FFMPEG_ENCODE_TIME'].match(line):
time_string = (
CFG_BIBENCODE_FFMPEG_ENCODE_TIME.match(line).groups()
cfg['CFG_BIBENCODE_FFMPEG_ENCODE_TIME'].match(
line).groups()
)[0]
break
filehandle.close()
Expand Down Expand Up @@ -521,9 +511,9 @@ def graphical(value):


## Run the encoding
pass_log_file = CFG_BIBENCODE_FFMPEG_PASSLOGFILE_PREFIX % (
os.path.splitext(os.path.split(input_file)[1])[0],
str(uuid.uuid4()))
pass_log_file = cfg['CFG_BIBENCODE_FFMPEG_PASSLOGFILE_PREFIX'] % (
os.path.splitext(os.path.split(input_file)[1])[0],
str(uuid.uuid4()))
no_error = True
## For every encoding pass to do
for apass in range(0, passes):
Expand Down Expand Up @@ -614,3 +604,4 @@ def get_res_for_weird_aspect(width, aspect, avail_res):
return [str(width) + 'x' + str(height)]
else:
return possible_res

41 changes: 15 additions & 26 deletions invenio/modules/encoder/extract.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,35 +17,23 @@
# along with Invenio; if not, write to the Free Software Foundation, Inc.,
# 59 Temple Place, Suite 330, Boston, MA 02111-1307, USA.

"""BibEncode frame extraction module.
"""
"""BibEncode frame extraction module."""

__revision__ = "$Id$"
import os

from invenio.modules.encoder.config import (
CFG_BIBENCODE_FFMPEG_EXTRACT_COMMAND,
)
from invenio.legacy.bibsched.bibtask import (
task_update_progress,
write_message
)
from invenio.modules.encoder.utils import (
timecode_to_seconds,
seconds_to_timecode,
is_timecode,
is_seconds,
normalize_string,
getval,
chose
)
from invenio.modules.encoder.metadata import (
ffprobe_metadata
)
import subprocess
import os

from invenio.base.globals import cfg
from invenio.legacy.bibsched.bibtask import task_update_progress, write_message
from invenio.modules.encoder.encode import \
determine_resolution_preserving_aspect
from invenio.modules.encoder.metadata import ffprobe_metadata
from invenio.modules.encoder.profiles import get_extract_profile
from invenio.modules.encoder.encode import determine_resolution_preserving_aspect
import re
from invenio.modules.encoder.utils import chose, is_seconds, is_timecode, \
seconds_to_timecode, timecode_to_seconds

__revision__ = "$Id$"


# rename size to resolution
def extract_frames(input_file, output_file=None, size=None, positions=None,
Expand Down Expand Up @@ -220,7 +208,7 @@ def extract_frames(input_file, output_file=None, size=None, positions=None,
#-------------#

## Build the command for ffmpeg
command = (CFG_BIBENCODE_FFMPEG_EXTRACT_COMMAND % (
command = (cfg['CFG_BIBENCODE_FFMPEG_EXTRACT_COMMAND'] % (
position, input_file, size, output_filename
)).split()
## Start subprocess and poll the output until it finishes
Expand All @@ -247,3 +235,4 @@ def extract_frames(input_file, output_file=None, size=None, positions=None,
## Everything should be fine if this position is reached
message_fnc("Extraction of frames was successful")
return 1

31 changes: 16 additions & 15 deletions invenio/modules/encoder/metadata.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,24 +23,23 @@

__revision__ = "$Id$"

import subprocess
import re
from six import iteritems
import subprocess
from invenio.base.globals import cfg
from invenio.legacy.bibsched.bibtask import write_message
from invenio.modules.encoder.utils import getval, mediainfo, probe
from invenio.utils.json import json, json_decode_file

from xml.dom import minidom

from invenio.utils.json import json, json_decode_file
from invenio.legacy.bibsched.bibtask import write_message
from invenio.modules.encoder.config import (
CFG_BIBENCODE_FFMPEG_METADATA_ARGUMENT,
CFG_BIBENCODE_FFMPEG_METADATA_SET_COMMAND,
CFG_BIBENCODE_PBCORE_MAPPINGS
)
from invenio.modules.encoder.utils import probe, getval, mediainfo, seconds_to_timecode
from six import iteritems


# Stores metadata for the process. Many different functions in BibEncode
# need access to video metadata regularly. Because we dont pass objects arount
# we need to call the functions of this submodule again and again. To not
# call ffprobe and mediainfo all the time, the metadata is stored in this cache.
# call ffprobe and mediainfo all the time, the metadata is stored in this
# cache.
_FFPROBE_METADATA_CACHE = {}
_MEDIAINFO_METADATA_CACHE = {}

Expand All @@ -57,12 +56,15 @@ def write_metadata(input_file, output_file, metadata):
## build metadata arguments for ffmpeg
for key, value in iteritems(metadata):
if value is not None:
meta_args.append(CFG_BIBENCODE_FFMPEG_METADATA_ARGUMENT % (key, value))
meta_args.append(
cfg['CFG_BIBENCODE_FFMPEG_METADATA_ARGUMENT'] % \
(key, value))
else:
write_message("metadata arg no dict")
return 0
## build the command
command = (CFG_BIBENCODE_FFMPEG_METADATA_SET_COMMAND % (input_file, output_file)).split()
command = (cfg['CFG_BIBENCODE_FFMPEG_METADATA_SET_COMMAND'] % \
(input_file, output_file)).split()
for meta_arg in meta_args:
command.insert(-1, '-metadata')
command.insert(-1, meta_arg)
Expand Down Expand Up @@ -317,7 +319,7 @@ def _map_values(mapping, locals_u, meta_dict, probe_dict, stream_number=None):
probe_dict = ffprobe_metadata(input_file)

# parse the mappings
pbcore_mappings = json_decode_file(CFG_BIBENCODE_PBCORE_MAPPINGS)
pbcore_mappings = json_decode_file(cfg['CFG_BIBENCODE_PBCORE_MAPPINGS'])

## INSTANTIATION ##
# According to the PBcore standard, this strict order MUST be followed
Expand Down Expand Up @@ -371,4 +373,3 @@ def _map_values(mapping, locals_u, meta_dict, probe_dict, stream_number=None):
joined = joined % {"xmlns" : ""}

return joined

Loading

0 comments on commit d2f5863

Please sign in to comment.