Skip to content

Commit

Permalink
add '--abort-on-skip' option and ability to control skip behavior
Browse files Browse the repository at this point in the history
the 'skip' config option controls skipping behavior:
    true    - skip download if file already exist (default)
    false   - download and overwrite files even if it exists
    "abort" - abort extractor run if a download would be skipped
              (same as '--abort-on-skip')
  • Loading branch information
mikf committed May 3, 2017
1 parent 7c8f61a commit fc9223c
Show file tree
Hide file tree
Showing 5 changed files with 37 additions and 15 deletions.
2 changes: 1 addition & 1 deletion gallery_dl/extractor/hbrowse.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ class HbrowseMangaExtractor(Extractor):
subcategory = "manga"
pattern = [r"(?:https?://)?(?:www\.)?hbrowse\.com/(\d+)/?$"]
test = [("http://www.hbrowse.com/10363", {
"url": "b89682bfb86c11d2af0dc47463804ec3ac4aadd6",
"url": "4d9def5df21c23f8c3d36de2076c189c02ea43bd",
})]

def __init__(self, match):
Expand Down
14 changes: 13 additions & 1 deletion gallery_dl/option.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,12 @@ def __call__(self, parser, namespace, values, option_string=None):
namespace.options.append(((self.dest,), values))


class ConfigConstAction(argparse.Action):
"""Set argparse const values as config values"""
def __call__(self, parser, namespace, values, option_string=None):
namespace.options.append(((self.dest,), self.const))


class ParseAction(argparse.Action):
"""Parse <key>=<value> options and set them as config values"""
def __call__(self, parser, namespace, values, option_string=None):
Expand Down Expand Up @@ -98,14 +104,20 @@ def build_parser():
metavar="ITEM-SPEC", action=ConfigAction, dest="chapters",
help=("Same as '--images' except for chapters")
)
parser.add_argument(
"--abort-on-skip",
action=ConfigConstAction, nargs=0, dest="skip", const="abort",
help=("Abort extractor run if a file download would normally be "
"skipped, i.e. if a file with the same filename already exists")
)
parser.add_argument(
"-R", "--retries",
metavar="RETRIES", action=ConfigAction, dest="retries", type=int,
help="Number of retries (default: 5)",
)
parser.add_argument(
"--http-timeout",
metavar="SECONDS", action=ConfigAction, dest="timeout", type=int,
metavar="SECONDS", action=ConfigAction, dest="timeout", type=float,
help="Timeout for HTTP connections (defaut: no timeout)",
)
parser.add_argument(
Expand Down
32 changes: 21 additions & 11 deletions gallery_dl/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -133,26 +133,27 @@ def __bool__(self):
class PathFormat():

def __init__(self, extractor):
key = ["extractor", extractor.category]
if extractor.subcategory:
key.append(extractor.subcategory)
self.filename_fmt = config.interpolate(
key + ["filename"], default=extractor.filename_fmt
)
self.directory_fmt = config.interpolate(
key + ["directory"], default=extractor.directory_fmt
)
self.filename_fmt = extractor.config(
"filename", extractor.filename_fmt)
self.directory_fmt = extractor.config(
"directory", extractor.directory_fmt)
self.has_extension = False
self.keywords = {}
self.directory = self.realdirectory = ""
self.path = self.realpath = ""

skipmode = extractor.config("skip", True)
if skipmode == "abort":
self.exists = self._exists_abort
elif not skipmode:
self.exists = self._exists_false

def open(self):
"""Open file ta 'realpath' and return a corresponding file object"""
"""Open file to 'realpath' and return a corresponding file object"""
return open(self.realpath, "wb")

def exists(self):
"""Return True if 'path' is complete and referse to an existing path"""
"""Return True if 'path' is complete and refers to an existing path"""
if self.has_extension:
return os.path.exists(self.realpath)
return False
Expand Down Expand Up @@ -189,6 +190,15 @@ def build_path(self, sep=os.path.sep):
self.path = self.directory + sep + filename
self.realpath = self.realdirectory + sep + filename

def _exists_abort(self):
if self.has_extension and os.path.exists(self.realpath):
raise exception.StopExtraction()
return False

@staticmethod
def _exists_false():
return False

@staticmethod
def get_base_directory():
"""Return the base-destination-directory for downloads"""
Expand Down
2 changes: 1 addition & 1 deletion gallery_dl/version.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,4 +6,4 @@
# it under the terms of the GNU General Public License version 2 as
# published by the Free Software Foundation.

__version__ = "0.8.3"
__version__ = "0.8.4-dev"
2 changes: 1 addition & 1 deletion test/test_extractors.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ def test(self):
# dont work on travis-ci
"exhentai", "kissmanga", "mangafox", "dynastyscans",
# temporary issues
"fallenangels",

]
# enable selective testing for direct calls
if __name__ == '__main__' and len(sys.argv) > 1:
Expand Down

0 comments on commit fc9223c

Please sign in to comment.