From 0e310d4021a223cf041272613d0f11f5c3c9b1cb Mon Sep 17 00:00:00 2001 From: George Kadianakis Date: Fri, 31 Jan 2020 16:28:53 +0100 Subject: [PATCH] Initial support for v3 onion services. See 'v3_dev_wip' branch for more fine grained details. --- .gitattributes | 1 + .travis.yml | 2 +- MANIFEST.in | 2 + onionbalance-config.py | 12 + onionbalance.py | 5 +- onionbalance/__init__.py | 4 + onionbalance/_version.py | 520 +++++ onionbalance/common/__init__.py | 0 onionbalance/common/argparser.py | 40 + onionbalance/common/descriptor.py | 40 + onionbalance/{ => common}/instance.py | 81 +- onionbalance/common/intro_point_set.py | 80 + onionbalance/{ => common}/log.py | 0 onionbalance/{ => common}/scheduler.py | 3 +- onionbalance/common/signalhandler.py | 36 + onionbalance/common/util.py | 69 + onionbalance/config_generator/__init__.py | 0 .../config_generator/config_generator.py | 434 ++++ .../config_generator/data/torrc-instance-v2 | 19 + .../data/torrc-instance-v3} | 0 .../{ => config_generator}/data/torrc-server | 0 onionbalance/data/config.example.yaml | 16 - onionbalance/hs_v2/__init__.py | 0 onionbalance/{ => hs_v2}/__main__.py | 2 +- onionbalance/{ => hs_v2}/config.py | 7 - onionbalance/{ => hs_v2}/consensus.py | 4 +- onionbalance/{ => hs_v2}/descriptor.py | 107 +- onionbalance/{ => hs_v2}/eventhandler.py | 58 +- onionbalance/hs_v2/instance.py | 87 + onionbalance/hs_v2/manager.py | 98 + onionbalance/{ => hs_v2}/service.py | 27 +- onionbalance/hs_v2/settings.py | 106 + onionbalance/{ => hs_v2}/status.py | 4 +- onionbalance/{ => hs_v2}/util.py | 12 +- onionbalance/hs_v3/__init__.py | 0 onionbalance/hs_v3/consensus.py | 274 +++ onionbalance/hs_v3/descriptor.py | 276 +++ onionbalance/hs_v3/hashring.py | 228 +++ onionbalance/hs_v3/instance.py | 80 + onionbalance/hs_v3/manager.py | 57 + onionbalance/hs_v3/onionbalance.py | 211 ++ onionbalance/hs_v3/params.py | 45 + onionbalance/hs_v3/service.py | 331 +++ onionbalance/hs_v3/stem_controller.py | 64 + onionbalance/hs_v3/tor_node.py | 75 + onionbalance/hs_v3/util.py | 3 + onionbalance/manager.py | 157 -- onionbalance/settings.py | 380 ---- setup.cfg | 10 +- setup.py | 16 +- test/functional/test_onionbalance_config.py | 10 +- .../test_publish_master_descriptor.py | 4 +- ...stall-chutney.sh => install-chutney-v2.sh} | 0 test/v2/__init__.py | 0 test/{ => v2}/test_consensus.py | 4 +- test/{ => v2}/test_descriptor.py | 18 +- test/{ => v2}/test_settings.py | 2 +- test/{ => v2}/test_util.py | 2 +- test/{ => v2}/util.py | 0 test/v3/test_v3_descriptor.py | 22 + test/v3/test_v3_hashring.py | 100 + versioneer.py | 1822 +++++++++++++++++ 62 files changed, 5233 insertions(+), 834 deletions(-) create mode 100644 .gitattributes create mode 100755 onionbalance-config.py create mode 100644 onionbalance/_version.py create mode 100644 onionbalance/common/__init__.py create mode 100644 onionbalance/common/argparser.py create mode 100644 onionbalance/common/descriptor.py rename onionbalance/{ => common}/instance.py (50%) create mode 100644 onionbalance/common/intro_point_set.py rename onionbalance/{ => common}/log.py (100%) rename onionbalance/{ => common}/scheduler.py (97%) create mode 100644 onionbalance/common/signalhandler.py create mode 100644 onionbalance/common/util.py create mode 100644 onionbalance/config_generator/__init__.py create mode 100644 onionbalance/config_generator/config_generator.py create mode 100644 onionbalance/config_generator/data/torrc-instance-v2 rename onionbalance/{data/torrc-instance => config_generator/data/torrc-instance-v3} (100%) rename onionbalance/{ => config_generator}/data/torrc-server (100%) delete mode 100644 onionbalance/data/config.example.yaml create mode 100644 onionbalance/hs_v2/__init__.py rename onionbalance/{ => hs_v2}/__main__.py (56%) rename onionbalance/{ => hs_v2}/config.py (80%) rename onionbalance/{ => hs_v2}/consensus.py (96%) rename onionbalance/{ => hs_v2}/descriptor.py (64%) rename onionbalance/{ => hs_v2}/eventhandler.py (52%) create mode 100644 onionbalance/hs_v2/instance.py create mode 100644 onionbalance/hs_v2/manager.py rename onionbalance/{ => hs_v2}/service.py (93%) create mode 100644 onionbalance/hs_v2/settings.py rename onionbalance/{ => hs_v2}/status.py (98%) rename onionbalance/{ => hs_v2}/util.py (93%) create mode 100644 onionbalance/hs_v3/__init__.py create mode 100644 onionbalance/hs_v3/consensus.py create mode 100644 onionbalance/hs_v3/descriptor.py create mode 100644 onionbalance/hs_v3/hashring.py create mode 100644 onionbalance/hs_v3/instance.py create mode 100644 onionbalance/hs_v3/manager.py create mode 100644 onionbalance/hs_v3/onionbalance.py create mode 100644 onionbalance/hs_v3/params.py create mode 100644 onionbalance/hs_v3/service.py create mode 100644 onionbalance/hs_v3/stem_controller.py create mode 100644 onionbalance/hs_v3/tor_node.py create mode 100644 onionbalance/hs_v3/util.py delete mode 100644 onionbalance/manager.py delete mode 100644 onionbalance/settings.py rename test/scripts/{install-chutney.sh => install-chutney-v2.sh} (100%) create mode 100644 test/v2/__init__.py rename test/{ => v2}/test_consensus.py (96%) rename test/{ => v2}/test_descriptor.py (97%) rename test/{ => v2}/test_settings.py (97%) rename test/{ => v2}/test_util.py (99%) rename test/{ => v2}/util.py (100%) create mode 100644 test/v3/test_v3_descriptor.py create mode 100644 test/v3/test_v3_hashring.py create mode 100644 versioneer.py diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..5d60251 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +onionbalance/_version.py export-subst diff --git a/.travis.yml b/.travis.yml index e397158..18fdb27 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,7 +9,7 @@ env: before_install: # Install tor and chutney if doing functional tests - if [[ $TEST == 'functional' ]]; then ./test/scripts/install-tor.sh; fi - - if [[ $TEST == 'functional' ]]; then source test/scripts/install-chutney.sh; fi + - if [[ $TEST == 'functional' ]]; then source test/scripts/install-chutney-v2.sh; fi install: - pip install tox coveralls script: diff --git a/MANIFEST.in b/MANIFEST.in index f6e5dc0..3e3e45a 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,3 +4,5 @@ include requirements.txt include tox.ini recursive-include docs *.rst recursive-include onionbalance/data * +include versioneer.py +include onionbalance/_version.py diff --git a/onionbalance-config.py b/onionbalance-config.py new file mode 100755 index 0000000..26942b0 --- /dev/null +++ b/onionbalance-config.py @@ -0,0 +1,12 @@ +#! /usr/bin/env python3 +# -*- coding: utf-8 -*- + +""" +Convenience wrapper for running onionbalance config generator directly from source tree. +""" + +import onionbalance.config_generator.config_generator + +if __name__ == '__main__': + onionbalance.config_generator.config_generator.main() + diff --git a/onionbalance.py b/onionbalance.py index e4e2109..1fdc990 100755 --- a/onionbalance.py +++ b/onionbalance.py @@ -3,7 +3,8 @@ """Convenience wrapper for running OnionBalance directly from source tree.""" -from onionbalance.manager import main +import onionbalance.hs_v2.manager +import onionbalance.hs_v3.manager if __name__ == '__main__': - main() + onionbalance.hs_v3.manager.main() diff --git a/onionbalance/__init__.py b/onionbalance/__init__.py index f2a3165..4d9119b 100644 --- a/onionbalance/__init__.py +++ b/onionbalance/__init__.py @@ -5,3 +5,7 @@ __contact__ = "donncha@donncha.is" __url__ = "https://github.com/DonnchaC/onionbalance" __license__ = "GPL" + +from ._version import get_versions +__version__ = get_versions()['version'] +del get_versions diff --git a/onionbalance/_version.py b/onionbalance/_version.py new file mode 100644 index 0000000..8b1b683 --- /dev/null +++ b/onionbalance/_version.py @@ -0,0 +1,520 @@ + +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +# This file is released into the public domain. Generated by +# versioneer-0.18 (https://github.com/warner/python-versioneer) + +"""Git implementation of _version.py.""" + +import errno +import os +import re +import subprocess +import sys + + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "$Format:%d$" + git_full = "$Format:%H$" + git_date = "$Format:%ci$" + keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "pep440" + cfg.tag_prefix = "" + cfg.parentdir_prefix = "onionbalance-" + cfg.versionfile_source = "onionbalance/_version.py" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None, None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None, None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + print("stdout was %s" % stdout) + return None, p.returncode + return stdout, p.returncode + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%s', no digits" % ",".join(refs - tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %s" % r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %s not under git control" % root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], + cwd=root)[0].strip() + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None} + + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", "date": None} diff --git a/onionbalance/common/__init__.py b/onionbalance/common/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/onionbalance/common/argparser.py b/onionbalance/common/argparser.py new file mode 100644 index 0000000..500d924 --- /dev/null +++ b/onionbalance/common/argparser.py @@ -0,0 +1,40 @@ +import argparse +import os + +import onionbalance + +TOR_CONTROL_SOCKET = os.environ.get('ONIONBALANCE_TOR_CONTROL_SOCKET', + '/var/run/tor/control') + +def get_common_argparser(): + """ + Parses and returns command line arguments. + """ + + parser = argparse.ArgumentParser( + description="onionbalance distributes the requests for a Tor hidden " + "services across multiple Tor instances.") + + parser.add_argument("-i", "--ip", type=str, default='127.0.0.1', + help="Tor controller IP address") + + parser.add_argument("-p", "--port", type=int, default=9051, + help="Tor controller port") + + parser.add_argument("-s", "--socket", type=str, default=TOR_CONTROL_SOCKET, + help="Tor unix domain control socket location") + + parser.add_argument("-c", "--config", type=str, + default=os.environ.get('ONIONBALANCE_CONFIG', + "config.yaml"), + help="Config file location") + + parser.add_argument("-v", "--verbosity", type=str, default=None, + help="Minimum verbosity level for logging. Available " + "in ascending order: debug, info, warning, " + "error, critical). The default is info.") + + parser.add_argument('--version', action='version', + version='onionbalance %s' % onionbalance.__version__) + + return parser diff --git a/onionbalance/common/descriptor.py b/onionbalance/common/descriptor.py new file mode 100644 index 0000000..89c7483 --- /dev/null +++ b/onionbalance/common/descriptor.py @@ -0,0 +1,40 @@ +import stem + +from onionbalance.common import log + +logger = log.get_logger() + +def upload_descriptor(controller, signed_descriptor, hsdirs=None, v3_onion_address=None): + """ + Upload descriptor via the Tor control port + + If no HSDirs are specified, Tor will upload to what it thinks are the + responsible directories + + If 'v3_onion_address' is set, this is a v3 HSPOST request, and the address + needs to be embedded in the request. + """ + logger.debug("Beginning service descriptor upload.") + + server_args = "" + + # Provide server fingerprints to control command if HSDirs are specified. + if hsdirs: + server_args = ' '.join([("SERVER={}".format(hsdir)) + for hsdir in hsdirs]) + + if v3_onion_address: + server_args += " HSADDRESS=%s" % v3_onion_address.replace(".onion","") + + # Stem will insert the leading + and trailing '\r\n.\r\n' + response = controller.msg("HSPOST %s\n%s" % + (server_args, signed_descriptor)) + + (response_code, divider, response_content) = response.content()[0] + if not response.is_ok(): + if response_code == "552": + raise stem.InvalidRequest(response_code, response_content) + else: + raise stem.ProtocolError("HSPOST returned unexpected response " + "code: %s\n%s" % (response_code, + response_content)) diff --git a/onionbalance/instance.py b/onionbalance/common/instance.py similarity index 50% rename from onionbalance/instance.py rename to onionbalance/common/instance.py index e12a8a1..f8d8717 100644 --- a/onionbalance/instance.py +++ b/onionbalance/common/instance.py @@ -1,17 +1,13 @@ -# -*- coding: utf-8 -*- -import datetime import time import stem.control -from onionbalance import log -from onionbalance import config -from onionbalance import util +from onionbalance.common import log +import onionbalance.common.util logger = log.get_logger() - -def fetch_instance_descriptors(controller): +def helper_fetch_all_instance_descriptors(controller, instances, control_password=None): """ Try fetch fresh descriptors for all HS instances """ @@ -25,14 +21,14 @@ def fetch_instance_descriptors(controller): # the NEWNYM singal controller.signal(stem.control.Signal.NEWNYM) time.sleep(5) # Sleep to allow Tor time to build new circuits + pass except stem.SocketClosed: logger.error("Failed to send NEWNYM signal, socket is closed.") - util.reauthenticate(controller, logger) + onionbalance.common.util.reauthenticate(controller, logger, control_password) else: break - unique_instances = set(instance for service in config.services - for instance in service.instances) + unique_instances = set(instances) # Only try to retrieve the descriptor once for each unique instance # address. An instance may be configured under multiple master @@ -45,13 +41,11 @@ def fetch_instance_descriptors(controller): try: instance.fetch_descriptor() except stem.SocketClosed: - logger.error("Failed to fecth descriptor, socket " - "is closed") - util.reauthenticate(controller, logger) + logger.error("Failed to fetch descriptor, socket is closed") + onionbalance.common.util.reauthenticate(controller, logger, control_password) else: break - class Instance(object): """ Instance represents a back-end load balancing hidden service. @@ -67,21 +61,11 @@ def __init__(self, controller, onion_address, authentication_cookie=None): if onion_address: onion_address = onion_address.replace('.onion', '') self.onion_address = onion_address - self.authentication_cookie = authentication_cookie - - # Store the latest set of introduction points for this instance - self.introduction_points = [] - - # Timestamp when last received a descriptor for this instance - self.received = None - - # Timestamp of the currently loaded descriptor - self.timestamp = None # Flag this instance with its introduction points change. A new # master descriptor will then be published as the introduction # points have changed. - self.changed_since_published = False + self.intro_set_changed_since_published = False def fetch_descriptor(self): """ @@ -101,53 +85,6 @@ def fetch_descriptor(self): logger.warning("No descriptor received for instance %s.onion, " "the instance may be offline.", self.onion_address) - def update_descriptor(self, parsed_descriptor): - """ - Update introduction points when a new HS descriptor is received - - Parse the descriptor content and update the set of introduction - points for this HS instance. Returns True if the introduction - point set has changed, False otherwise.` - """ - - self.received = datetime.datetime.utcnow() - - logger.debug("Received a descriptor for instance %s.onion.", - self.onion_address) - - # Reject descriptor if its timestamp is older than the current - # descriptor. Prevents HSDirs from replaying old, expired - # descriptors. - if self.timestamp and parsed_descriptor.published < self.timestamp: - logger.error("Received descriptor for instance %s.onion with " - "publication timestamp (%s) older than the latest " - "descriptor (%s). Ignoring the descriptor.", - self.onion_address, - parsed_descriptor.published, - self.timestamp) - return False - else: - self.timestamp = parsed_descriptor.published - - # Parse the introduction point list, decrypting if necessary - introduction_points = parsed_descriptor.introduction_points( - authentication_cookie=self.authentication_cookie - ) - - # If the new introduction points are different, flag this instance - # as modified. Compare the set of introduction point identifiers - # (fingerprint of the per IP circuit service key). - if (set(ip.identifier for ip in introduction_points) != - set(ip.identifier for ip in self.introduction_points)): - self.changed_since_published = True - self.introduction_points = introduction_points - return True - - else: - logger.debug("Introduction points for instance %s.onion matched " - "the cached set.", self.onion_address) - return False - def __eq__(self, other): """ Instance objects are equal if they have the same onion address. diff --git a/onionbalance/common/intro_point_set.py b/onionbalance/common/intro_point_set.py new file mode 100644 index 0000000..8358cb5 --- /dev/null +++ b/onionbalance/common/intro_point_set.py @@ -0,0 +1,80 @@ +from future.moves.itertools import zip_longest +import random +import itertools + +class IntroductionPointSet(object): + """ + A set of introduction points to included in a HS descriptor. + + Provided with a list of available introduction points for each backend + instance for an onionbalance service. This object will store the set of + available introduction points and allow IPs to be selected from the + available set. + + This class tracks which introduction points have already been provided + and tries to provide the most diverse set of IPs. + """ + def __init__(self, intro_points): + """ + 'intro_points' is a list of lists that looks like this: + [ + [], + [], + [], + ... + ] + """ + # Shuffle the introduction point order before selecting IPs. + # Randomizing now allows later calls to .choose() to be + # deterministic. + for instance_intro_points in intro_points: + random.shuffle(instance_intro_points) + random.shuffle(intro_points) + + self.intro_points = intro_points + self._intro_point_generator = self._get_intro_point() + + def __len__(self): + """Provide the total number of available introduction points""" + return sum(len(ips) for ips in self.intro_points) + + def _get_intro_point(self): + """ + [Private function] + + Generator function which yields an introduction point + + Iterates through all available introduction points and try + to pick IPs breath first across all backend instances. The + intro point set is wrapped in `itertools.cycle` and will provided + an infinite series of introduction points. + """ + + # Combine intro points from across the backend instances and flatten + intro_points = zip_longest(*self.intro_points) + flat_intro_points = itertools.chain.from_iterable(intro_points) + for intro_point in itertools.cycle(flat_intro_points): + if intro_point: + yield intro_point + + def choose(self, count=10, shuffle=True): + """ + [Public API] + + Retrieve N introduction points from the set of IPs + + Where more than `count` IPs are available, introduction points are + selected to try and achieve the greatest distribution of introduction + points across all of the available backend instances. + + Return a list of IntroductionPoints. + """ + + # Limit `count` to the available number of IPs to avoid repeats. + count = min(len(self), count) + choosen_ips = list(itertools.islice(self._intro_point_generator, count)) + + if shuffle: + random.shuffle(choosen_ips) + return choosen_ips + diff --git a/onionbalance/log.py b/onionbalance/common/log.py similarity index 100% rename from onionbalance/log.py rename to onionbalance/common/log.py diff --git a/onionbalance/scheduler.py b/onionbalance/common/scheduler.py similarity index 97% rename from onionbalance/scheduler.py rename to onionbalance/common/scheduler.py index 195fea7..41f8dc7 100644 --- a/onionbalance/scheduler.py +++ b/onionbalance/common/scheduler.py @@ -5,7 +5,7 @@ import functools import time -from onionbalance import log +from onionbalance.common import log logger = log.get_logger() @@ -42,7 +42,6 @@ def run(self, override_run_time=None): """ Run job then reschedule it in the job list """ - logger.debug("Running {}".format(self)) ret = self.job_func() # Pretend the job was scheduled now, if we ran it early with run_all() diff --git a/onionbalance/common/signalhandler.py b/onionbalance/common/signalhandler.py new file mode 100644 index 0000000..ffb319f --- /dev/null +++ b/onionbalance/common/signalhandler.py @@ -0,0 +1,36 @@ +import sys +import signal +import logging + +from onionbalance.common import log + +logger = log.get_logger() + +class SignalHandler(object): + """ + Handle signals sent to the OnionBalance daemon process + """ + + def __init__(self, controller, status_socket=None): + """ + Setup signal handler + """ + self._tor_controller = controller + self._status_socket = status_socket + + # Register signal handlers + signal.signal(signal.SIGTERM, self._handle_sigint_sigterm) + signal.signal(signal.SIGINT, self._handle_sigint_sigterm) + + def _handle_sigint_sigterm(self, signum, frame): + """ + Handle SIGINT (Ctrl-C) and SIGTERM + + Disconnect from control port and cleanup the status socket + """ + logger.info("Signal %d received, exiting", signum) + self._tor_controller.close() + if self._status_socket: + self._status_socket.close() + logging.shutdown() + sys.exit(0) diff --git a/onionbalance/common/util.py b/onionbalance/common/util.py new file mode 100644 index 0000000..121d098 --- /dev/null +++ b/onionbalance/common/util.py @@ -0,0 +1,69 @@ +import sys +import time +import os +import yaml +import stem + +from stem.control import Controller + +from onionbalance.common import log + +logger = log.get_logger() + +def read_config_data_from_file(config_path): + if os.path.exists(config_path): + with open(config_path, 'r') as handle: + config_data = yaml.safe_load(handle.read()) + logger.info("Loaded the config file '%s'.", config_path) + else: + logger.error("The specified config file '%s' does not exist. The " + "onionbalance-config tool can generate the required " + "keys and config files.", config_path) + sys.exit(1) + + return config_data + +def connect_to_control_port(tor_socket=None, tor_address=None, tor_port=0, control_password=None): + controller = None + + # Try first with a connection to the Tor unix domain control socket + if tor_socket: + try: + controller = Controller.from_socket_file(path=tor_socket) + logger.debug("Successfully connected to the Tor control socket " + "%s.", tor_socket) + except stem.SocketError: + logger.debug("Unable to connect to the Tor control socket %s.", + tor_socket) + + # If we didn't manage to connect to control socket, try IP:PORT + if not controller: + try: + controller = Controller.from_port(address=tor_address, + port=tor_port) + logger.debug("Successfully connected to the Tor control port.") + except stem.SocketError as exc: + logger.error("Unable to connect to Tor control port: %s", exc) + sys.exit(1) + + try: + controller.authenticate(password=control_password) + except stem.connection.AuthenticationFailure as exc: + logger.error("Unable to authenticate on the Tor control connection: " + "%s", exc) + sys.exit(1) + else: + logger.debug("Successfully authenticated on the Tor control " + "connection.") + + return controller + +def reauthenticate(controller, logger, control_password=None): + """ + Tries to authenticate to the controller + """ + time.sleep(10) + try: + controller.authenticate(password=control_password) + except stem.connection.AuthenticationFailure: + logger.error("Failed to re-authenticate controller.") diff --git a/onionbalance/config_generator/__init__.py b/onionbalance/config_generator/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/onionbalance/config_generator/config_generator.py b/onionbalance/config_generator/config_generator.py new file mode 100644 index 0000000..048affa --- /dev/null +++ b/onionbalance/config_generator/config_generator.py @@ -0,0 +1,434 @@ +import argparse +import logging +import os +import sys +import getpass +import yaml +import pkg_resources + +import Crypto.PublicKey.RSA +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from cryptography.hazmat.primitives import serialization + +from stem.descriptor.hidden_service import HiddenServiceDescriptorV3 + +import onionbalance +from onionbalance.common import log +from onionbalance.hs_v2 import util + +# Simplify the logging output for the command line tool +logger = log.get_config_generator_logger() + +class ConfigGenerator(object): + def __init__(self, args, interactive): + self.args = args + self.interactive = interactive + + self.hs_version = None + self.output_path = None + self.master_key = None + self.master_onion_address = None + self.num_instances = None + self.tag = None + self.torrc_port_line = None + self.instances = None + self.master_dir = None + + # Gather information required to create config file! + self.gather_information() + + # Create config file! + self.generate_config() + + def gather_information(self): + self.hs_version = self.get_hs_version() + assert(self.hs_version in ['v2','v3']) + + # Check if output directory exists, if not try create it + self.output_path = self.get_output_path() + self.master_dir = os.path.join(self.output_path, 'master') + + # Load the master key + self.master_key, self.master_onion_address = self.load_master_key() + + # Finished loading/generating master key, now try generate keys for + # each service instance + self.num_instances, self.tag = self.get_num_instances() + + # Create HiddenServicePort line for instance torrc file + if self.hs_version == 'v2': + self.torrc_port_line = self.get_torrc_port_line() + + self.instances = self.create_instances() + + def generate_config(self): + self.write_master_key_to_disk() + + assert(self.instances) + self.create_yaml_config_file() + + # Generate config files for each service instance + if self.hs_version == 'v2': + self.write_v2_instance_files() + + def get_output_path(self): + """ + Get path to output directory and create if needed + """ + output_path = None + if self.interactive: + output_path = input("Enter path to store generated config " + "[{}]: ".format(os.path.abspath(self.args.output))) + output_path = output_path or self.args.output + try: + util.try_make_dir(output_path) + except OSError: + logger.exception("Problem encountered when trying to create the " + "output directory %s.", os.path.abspath(output_path)) + else: + logger.debug("Created the output directory '%s'.", + os.path.abspath(output_path)) + + # The output directory should be empty to avoid having conflict keys + # or config files. + if not util.is_directory_empty(output_path): + logger.error("The specified output directory is not empty. Please " + "delete any files and folders or specify another output " + "directory.") + sys.exit(1) + + return output_path + + def get_hs_version(self): + # Get the HS version + hs_version = None + if self.interactive: + hs_version = input('Enter HS version ("v2" or "v3") (Leave empty for "v3"): ') + hs_version = hs_version or self.args.hs_version + + if hs_version not in ["v2", "v3"]: + logger.error('Only accepting "v2" and "v3" as HS versions') + sys.exit(1) + + logger.info("Rolling with HS %s!", hs_version) + + return hs_version + + def load_master_key(self): + master_key_path = self.get_master_key_path() + + # master_key_path is now either None (if no key path is specified) or + # set to the actual path + if self.hs_version == 'v2': + return self.load_v2_master_key(master_key_path) + else: + return self.load_v3_master_key(master_key_path) + + def get_master_key_path(self): + # Load master key if specified + master_key_path = None + if self.interactive: + # Read key path from user + master_key_path = input("Enter path to master service private key " + "(Leave empty to generate a key): ") + master_key_path = self.args.key or master_key_path + + # If a key path was specified make sure it exists + if master_key_path: + if not os.path.isfile(master_key_path): + logger.error("The specified master service private key '%s' " + "could not be found. Please confirm the path and " + "file permissions are correct.", master_key_path) + sys.exit(1) + + return master_key_path + + + def load_v3_master_key(self, master_key_path): + if master_key_path: + # load key + raise NotImplementedError + else: + master_private_key = Ed25519PrivateKey.generate() + master_public_key = master_private_key.public_key() + master_pub_key_bytes = master_public_key.public_bytes(encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw) + master_onion_address = HiddenServiceDescriptorV3.address_from_identity_key(master_pub_key_bytes) + # cut out the onion since that's what the rest of the code expects + master_onion_address = master_onion_address.replace(".onion", "") + + return master_private_key, master_onion_address + + def load_v2_master_key(self, master_key_path): + if master_key_path: + # Try load the specified private key file + master_key = util.key_decrypt_prompt(master_key_path) + if not master_key: + logger.error("The specified master private key %s could not " + "be loaded.", os.path.abspath(master_key)) + sys.exit(1) + else: + master_onion_address = util.calc_onion_address(master_key) + logger.info("Successfully loaded a master key for service " + "%s.onion.", master_onion_address) + + else: + # No key specified, begin generating a new one. + master_key = Crypto.PublicKey.RSA.generate(1024) + master_onion_address = util.calc_onion_address(master_key) + logger.debug("Created a new master key for service %s.onion.", + master_onion_address) + + return master_key, master_onion_address + + def get_num_instances(self): + """ + Get the number of instances and a tag name for them. + """ + num_instances = None + if self.interactive: + num_instances = input("Number of instance services to create " + "[{}]: ".format(self.args.num_instances)) + # Cast to int if a number was specified + try: + num_instances = int(num_instances) + except ValueError: + num_instances = None + num_instances = num_instances or self.args.num_instances + logger.debug("Creating %d service instances.", num_instances) + + tag = None + if self.interactive: + tag = input("Provide a tag name to group these instances " + "[{}]: ".format(self.args.tag)) + tag = tag or self.args.tag + + return num_instances, tag + + def get_torrc_port_line(self): + """ + Get the HiddenServicePort line for the instance torrc file + """ + service_virtual_port = None + if self.interactive: + service_virtual_port = input("Specify the service virtual port (for " + "client connections) [{}]: ".format( + self.args.service_virtual_port)) + service_virtual_port = service_virtual_port or self.args.service_virtual_port + + service_target = None + if self.interactive: + # In interactive mode, change default target to match the specified + # virtual port + default_service_target = u'127.0.0.1:{}'.format(service_virtual_port) + service_target = input("Specify the service target IP and port (where " + "your service is listening) [{}]: ".format( + default_service_target)) + service_target = service_target or default_service_target + service_target = service_target or self.args.service_target + torrc_port_line = u'HiddenServicePort {} {}'.format(service_virtual_port, + service_target) + return torrc_port_line + + def create_instances(self): + if self.hs_version == 'v2': + return self.create_v2_instances() + else: + return self.create_v3_instances() + + def create_v2_instances(self): + instances = [] + + for i in range(0, self.num_instances): + instance_key = Crypto.PublicKey.RSA.generate(1024) + instance_address = util.calc_onion_address(instance_key) + logger.debug("Created a key for instance %s.onion.", + instance_address) + instances.append((instance_address, instance_key)) + + return instances + + def create_v3_instances(self): + instances = [] + + for i in range(0, self.num_instances): + instances.append(("", None)) + + return instances + + def get_master_key_passphrase(self): + # Get optional passphrase for master key + # [TODO: Implement for v3] + master_passphrase = None + if self.interactive: + master_passphrase = getpass.getpass( + "Provide an optional password to encrypt the master private " + "key (Not encrypted if no password is specified): ") + return master_passphrase or self.args.password + + def write_master_key_to_disk(self): + # Finished reading input, starting to write config files. + util.try_make_dir(self.master_dir) + master_key_file = os.path.join(self.master_dir, + '{}.key'.format(self.master_onion_address)) + with open(master_key_file, "wb") as key_file: + os.chmod(master_key_file, 384) # chmod 0600 in decimal + + if self.hs_version == 'v2': + master_passphrase = self.get_master_key_passphrase() + key_file.write(self.master_key.exportKey(passphrase=master_passphrase)) + else: + master_key_formatted = self.master_key.private_bytes(encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption()) + key_file.write(master_key_formatted) + + logger.debug("Successfully wrote master key to file %s.", + os.path.abspath(master_key_file)) + + def write_v2_instance_files(self): + for i, (instance_address, instance_key) in enumerate(self.instances): + # Create a numbered directory for instance + instance_dir = os.path.join(self.output_path, '{}{}'.format(self.tag, i+1)) + instance_key_dir = os.path.join(instance_dir, instance_address) + util.try_make_dir(instance_key_dir) + os.chmod(instance_key_dir, 1472) # chmod 2700 in decimal + + instance_key_file = os.path.join(instance_key_dir, 'private_key') + with open(instance_key_file, "wb") as key_file: + os.chmod(instance_key_file, 384) # chmod 0600 in decimal + key_file.write(instance_key.exportKey()) + logger.debug("Successfully wrote key for instance %s.onion to " + "file.", instance_address) + + # Write torrc file for each instance + instance_torrc = os.path.join(instance_dir, 'instance_torrc') + instance_torrc_template = pkg_resources.resource_string( + __name__, 'data/torrc-instance-v2') + with open(instance_torrc, "w") as torrc_file: + torrc_file.write(instance_torrc_template.decode('utf-8')) + # The ./ relative path prevents Tor from raising relative + # path warnings. The relative path may need to be edited manual + # to work on Windows systems. + torrc_file.write(u"HiddenServiceDir {}\n".format( + instance_address)) + torrc_file.write(u"{}\n".format(self.torrc_port_line)) + + def create_yaml_config_file(self): + # Create YAML OnionBalance settings file for these instances + service_data = {'key': '{}.key'.format(self.master_onion_address)} + service_data['instances'] = [{'address': address, + 'name': '{}{}'.format(self.tag, i+1)} for + i, (address, _) in enumerate(self.instances)] + settings_data = {'services': [service_data]} + config_yaml = yaml.safe_dump(settings_data, default_flow_style=False) + + config_file_path = os.path.join(self.master_dir, 'config.yaml') + with open(config_file_path, "w") as config_file: + config_file.write(u"# OnionBalance Config File\n") + config_file.write(config_yaml) + logger.info("Wrote master service config file '%s'.", + os.path.abspath(config_file_path)) + + # Write master service torrc + master_torrc_path = os.path.join(self.master_dir, 'torrc-server') + master_torrc_template = pkg_resources.resource_string(__name__, + 'data/torrc-server') + with open(master_torrc_path, "w") as master_torrc_file: + master_torrc_file.write(master_torrc_template.decode('utf-8')) + + +def parse_cmd_args(): + """ + Parses and returns command line arguments for config generator + """ + + parser = argparse.ArgumentParser( + description="onionbalance-config generates config files and keys for " + "OnionBalance instances and management servers. Calling without any " + "options will initiate an interactive mode.") + + parser.add_argument("--hs-version", type=str, + default="v3", + help="Onion service version (default: %(default)s).") + + parser.add_argument("--key", type=str, default=None, + help="RSA private key for the master onion service.") + + parser.add_argument("-p", "--password", type=str, default=None, + help="Optional password which can be used to encrypt" + "the master service private key.") + + parser.add_argument("-n", type=int, default=2, dest="num_instances", + help="Number of instances to generate (default: " + "%(default)s).") + + parser.add_argument("-t", "--tag", type=str, default='node', + help="Prefix name for the service instances " + "(default: %(default)s).") + + parser.add_argument("--output", type=str, default='config/', + help="Directory to store generate config files. " + "The directory will be created if it does not " + "already exist.") + + parser.add_argument("--no-interactive", action='store_true', + help="Try to run automatically without prompting for" + "user input.") + + parser.add_argument("-v", type=str, default="info", dest='verbosity', + help="Minimum verbosity level for logging. Available " + "in ascending order: debug, info, warning, error, " + "critical). The default is info.") + + parser.add_argument("--service-virtual-port", type=str, + default="80", + help="Onion service port for external client " + "connections (default: %(default)s).") + + # TODO: Add validator to check if the target host:port line makes sense. + parser.add_argument("--service-target", type=str, + default="127.0.0.1:80", + help="Target IP and port where your service is " + "listening (default: %(default)s).") + + # .. todo:: Add option to specify HS host and port for instance torrc + + parser.add_argument('--version', action='version', + version='onionbalance %s' % onionbalance.__version__) + + return parser + +def main(): + """ + Entry point for interactive config file generation. + """ + + # Parse initial command line options + args = parse_cmd_args().parse_args() + + logger.info("Beginning OnionBalance config generation.") + + # If CLI options have been provided, don't enter interactive mode + # Crude check to see if any options beside --verbosity are set. + verbose = True if '-v' in sys.argv else False + + if ((len(sys.argv) > 1 and not verbose) or len(sys.argv) > 3 or + args.no_interactive): + interactive = False + logger.info("Entering non-interactive mode.") + else: + interactive = True + logger.info("No command line arguments found, entering interactive " + "mode.") + + logger.setLevel(logging.__dict__[args.verbosity.upper()]) + + # Start the config generator! + config_generator = ConfigGenerator(args, interactive) + + logger.info("Done! Successfully generated an OnionBalance config and %d " + "instance keys for service %s.onion.", + config_generator.num_instances, config_generator.master_onion_address) + + sys.exit(0) diff --git a/onionbalance/config_generator/data/torrc-instance-v2 b/onionbalance/config_generator/data/torrc-instance-v2 new file mode 100644 index 0000000..9003d22 --- /dev/null +++ b/onionbalance/config_generator/data/torrc-instance-v2 @@ -0,0 +1,19 @@ +# Tor config for the onion service instance servers +# --- +# The instance servers run standard onion services. In Basic mode the +# control port does not need to be enabled. + +DataDirectory tor-data + +# ControlPort 9051 +# CookieAuthentication 1 +SocksPort 0 + +RunAsDaemon 1 + +HiddenServiceVersion 2 + +# Configure each onion service instance with a unique permanent key. +# HiddenServiceDir tor-data/hidden_service/ +# HiddenServicePort 80 127.0.0.1:80 + diff --git a/onionbalance/data/torrc-instance b/onionbalance/config_generator/data/torrc-instance-v3 similarity index 100% rename from onionbalance/data/torrc-instance rename to onionbalance/config_generator/data/torrc-instance-v3 diff --git a/onionbalance/data/torrc-server b/onionbalance/config_generator/data/torrc-server similarity index 100% rename from onionbalance/data/torrc-server rename to onionbalance/config_generator/data/torrc-server diff --git a/onionbalance/data/config.example.yaml b/onionbalance/data/config.example.yaml deleted file mode 100644 index 0401160..0000000 --- a/onionbalance/data/config.example.yaml +++ /dev/null @@ -1,16 +0,0 @@ -# Onion Load Balancer Config File -# --- -# Each hidden service key line should be followed be followed by a list of 0 -# or more instances which contain the onion address of the load balancing -# hidden service - -REFRESH_INTERVAL: 600 # How often to poll for updated descriptors -services: - - key: /path/to/private_key # 7s4hxwwifcslrus2.onion - instances: - - address: o6ff73vmigi4oxka # web1 - - address: nkz23ai6qesuwqhc # web2 - - key: /path/to/private_key.enc # dpkdeys3apjtqydk.onion - instances: - - address: htbzowpp5cn7wj2u # irc1 - - address: huey7aiod8dja8a3 # irc2 diff --git a/onionbalance/hs_v2/__init__.py b/onionbalance/hs_v2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/onionbalance/__main__.py b/onionbalance/hs_v2/__main__.py similarity index 56% rename from onionbalance/__main__.py rename to onionbalance/hs_v2/__main__.py index 120122b..250349d 100644 --- a/onionbalance/__main__.py +++ b/onionbalance/hs_v2/__main__.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -from onionbalance.manager import main +from onionbalance.hs_v2.hs_v2.manager import main if __name__ == "__main__": diff --git a/onionbalance/config.py b/onionbalance/hs_v2/config.py similarity index 80% rename from onionbalance/config.py rename to onionbalance/hs_v2/config.py index ad43582..72e9eba 100644 --- a/onionbalance/config.py +++ b/onionbalance/hs_v2/config.py @@ -10,7 +10,6 @@ REPLICAS = 2 HSDIR_SET = 3 # Publish each descriptor to 3 consecutive HSDirs MAX_INTRO_POINTS = 10 -DESCRIPTOR_VALIDITY_PERIOD = 24 * 60 * 60 DESCRIPTOR_OVERLAP_PERIOD = 60 * 60 DESCRIPTOR_UPLOAD_PERIOD = 60 * 60 # Re-upload descriptor every hour REFRESH_INTERVAL = 10 * 60 @@ -23,12 +22,6 @@ STATUS_SOCKET_LOCATION = os.environ.get('ONIONBALANCE_STATUS_SOCKET_LOCATION', '/var/run/onionbalance/control') -TOR_ADDRESS = '127.0.0.1' -TOR_PORT = 9051 -TOR_CONTROL_PASSWORD = None -TOR_CONTROL_SOCKET = os.environ.get('ONIONBALANCE_TOR_CONTROL_SOCKET', - '/var/run/tor/control') - # Upload multiple distinct descriptors containing different subsets of # the available introduction points DISTINCT_DESCRIPTORS = True diff --git a/onionbalance/consensus.py b/onionbalance/hs_v2/consensus.py similarity index 96% rename from onionbalance/consensus.py rename to onionbalance/hs_v2/consensus.py index a3abb6f..bc3a9bd 100644 --- a/onionbalance/consensus.py +++ b/onionbalance/hs_v2/consensus.py @@ -9,8 +9,8 @@ import stem import stem.descriptor -import onionbalance.log as log -import onionbalance.config as config +import onionbalance.common.log as log +import onionbalance.hs_v2.config as config logger = log.get_logger() diff --git a/onionbalance/descriptor.py b/onionbalance/hs_v2/descriptor.py similarity index 64% rename from onionbalance/descriptor.py rename to onionbalance/hs_v2/descriptor.py index c6fdbe1..18274f2 100644 --- a/onionbalance/descriptor.py +++ b/onionbalance/hs_v2/descriptor.py @@ -1,86 +1,18 @@ # -*- coding: utf-8 -*- -from future.moves.itertools import zip_longest import hashlib import base64 import textwrap import datetime -import random -import itertools import Crypto.Util.number -import stem +import stem.descriptor.hidden_service_descriptor -from onionbalance import util -from onionbalance import log -from onionbalance import config +from onionbalance.hs_v2 import util -logger = log.get_logger() - - -class IntroductionPointSet(object): - """ - Select a set of introduction points to included in a HS descriptor. - - Provided with a list of available introduction points for each - backend instance for an onionbalance service. This object will store - the set of available introduction points and allow IPs to be selected - from the available set. - - This class tracks which introduction points have already been provided - and tries to provide the most diverse set of IPs. - """ - - def __init__(self, available_introduction_points): - # Shuffle the introduction point order before selecting IPs. - # Randomizing now allows later calls to .choose() to be - # deterministic. - for instance_intro_points in available_introduction_points: - random.shuffle(instance_intro_points) - random.shuffle(available_introduction_points) - - self.available_intro_points = available_introduction_points - self.intro_point_generator = self.get_intro_point() - - def __len__(self): - """Provide the total number of available introduction points""" - return sum(len(ips) for ips in self.available_intro_points) - - def get_intro_point(self): - """ - Generator function which yields an introduction point - - Iterates through all available introduction points and try - to pick IPs breath first across all backend instances. The - intro point set is wrapped in `itertools.cycle` and will provided - an infinite series of introduction points. - """ - - # Combine intro points from across the backend instances and flatten - intro_points = zip_longest(*self.available_intro_points) - flat_intro_points = itertools.chain.from_iterable(intro_points) - for intro_point in itertools.cycle(flat_intro_points): - if intro_point: - yield intro_point - - def choose(self, count=10, shuffle=True): - """ - Retrieve N introduction points from the set of IPs - - Where more than `count` IPs are available, introduction points are - selected to try and achieve the greatest distribution of introduction - points across all of the available backend instances. - - Return a list of IntroductionPoints. - """ - - # Limit `count` to the available number of IPs to avoid repeats. - count = min(len(self), count) - choosen_ips = list(itertools.islice(self.intro_point_generator, count)) - - if shuffle: - random.shuffle(choosen_ips) - return choosen_ips +from onionbalance.common import log +from onionbalance.hs_v2 import config +logger = log.get_logger() def generate_service_descriptor(permanent_key, introduction_point_list=None, replica=0, timestamp=None, deviation=0): @@ -275,32 +207,3 @@ def descriptor_received(descriptor_content): return None - -def upload_descriptor(controller, signed_descriptor, hsdirs=None): - """ - Upload descriptor via the Tor control port - - If no HSDirs are specified, Tor will upload to what it thinks are the - responsible directories - """ - logger.debug("Beginning service descriptor upload.") - - # Provide server fingerprints to control command if HSDirs are specified. - if hsdirs: - server_args = ' '.join([("SERVER={}".format(hsdir)) - for hsdir in hsdirs]) - else: - server_args = "" - - # Stem will insert the leading + and trailing '\r\n.\r\n' - response = controller.msg("HSPOST %s\n%s" % - (server_args, signed_descriptor)) - - (response_code, divider, response_content) = response.content()[0] - if not response.is_ok(): - if response_code == "552": - raise stem.InvalidRequest(response_code, response_content) - else: - raise stem.ProtocolError("HSPOST returned unexpected response " - "code: %s\n%s" % (response_code, - response_content)) diff --git a/onionbalance/eventhandler.py b/onionbalance/hs_v2/eventhandler.py similarity index 52% rename from onionbalance/eventhandler.py rename to onionbalance/hs_v2/eventhandler.py index 182cf2d..3d79e9e 100644 --- a/onionbalance/eventhandler.py +++ b/onionbalance/hs_v2/eventhandler.py @@ -1,14 +1,12 @@ # -*- coding: utf-8 -*- from builtins import str, object -import logging -import signal -import sys import stem -from onionbalance import log -from onionbalance import descriptor -from onionbalance import consensus +from onionbalance.common import log + +from onionbalance.hs_v2 import descriptor +from onionbalance.hs_v2 import consensus logger = log.get_logger() @@ -22,18 +20,17 @@ class EventHandler(object): @staticmethod def new_status(status_event): """ - Parse Tor status events such as "STATUS_GENERAL" + Parse Tor status events such as "STATUS_CLIENT" """ # pylint: disable=no-member - if status_event.status_type == stem.StatusType.GENERAL: - if status_event.action == "CONSENSUS_ARRIVED": - # Update the local view of the consensus in OnionBalance - try: - consensus.refresh_consensus() - except Exception: - logger.exception("An unexpected exception occured in the " - "when processing the consensus update " - "callback.") + if status_event.action == "CONSENSUS_ARRIVED": + # Update the local view of the consensus in OnionBalance + try: + consensus.refresh_consensus() + except Exception: + logger.exception("An unexpected exception occured in the " + "when processing the consensus update " + "callback.") @staticmethod def new_desc(desc_event): @@ -71,32 +68,3 @@ def new_desc_content(desc_content_event): "new descriptor callback.") return None - - -class SignalHandler(object): - """ - Handle signals sent to the OnionBalance daemon process - """ - - def __init__(self, controller, status_socket): - """ - Setup signal handler - """ - self._tor_controller = controller - self._status_socket = status_socket - - # Register signal handlers - signal.signal(signal.SIGTERM, self._handle_sigint_sigterm) - signal.signal(signal.SIGINT, self._handle_sigint_sigterm) - - def _handle_sigint_sigterm(self, signum, frame): - """ - Handle SIGINT (Ctrl-C) and SIGTERM - - Disconnect from control port and cleanup the status socket - """ - logger.info("Signal %d received, exiting", signum) - self._tor_controller.close() - self._status_socket.close() - logging.shutdown() - sys.exit(0) diff --git a/onionbalance/hs_v2/instance.py b/onionbalance/hs_v2/instance.py new file mode 100644 index 0000000..e643828 --- /dev/null +++ b/onionbalance/hs_v2/instance.py @@ -0,0 +1,87 @@ +# -*- coding: utf-8 -*- +import datetime +import time + +from onionbalance.common import log +from onionbalance.hs_v2 import config +import onionbalance.common.instance + +from onionbalance.hs_v2 import util + +logger = log.get_logger() + +def fetch_instance_descriptors(controller): + all_instances = [instance for service in config.services for instance in service.instances] + + onionbalance.common.instance.helper_fetch_all_instance_descriptors(controller, all_instances, + control_password=config.TOR_CONTROL_PASSWORD) + +class InstanceV2(onionbalance.common.instance.Instance): + """ + This is a V2 onionbalance instance + """ + def __init__(self, controller, onion_address, authentication_cookie): + """ + Constructor for V2 instance + """ + # Initialize the common instance class + super().__init__(controller, onion_address) + + self.authentication_cookie = authentication_cookie + + # Store the latest set of introduction points for this instance + self.introduction_points = [] + + # Timestamp when last received a descriptor for this instance + self.received = None + + # Timestamp of the currently loaded descriptor + self.timestamp = None + + def update_descriptor(self, parsed_descriptor): + """ + Update introduction points when a new HS descriptor is received + + Parse the descriptor content and update the set of introduction + points for this HS instance. Returns True if the introduction + point set has changed, False otherwise.` + """ + + self.received = datetime.datetime.utcnow() + + logger.debug("Received a descriptor for instance %s.onion.", + self.onion_address) + + # Reject descriptor if its timestamp is older than the current + # descriptor. Prevents HSDirs from replaying old, expired + # descriptors. + if self.timestamp and parsed_descriptor.published < self.timestamp: + logger.error("Received descriptor for instance %s.onion with " + "publication timestamp (%s) older than the latest " + "descriptor (%s). Ignoring the descriptor.", + self.onion_address, + parsed_descriptor.published, + self.timestamp) + return False + else: + self.timestamp = parsed_descriptor.published + + # Parse the introduction point list, decrypting if necessary + introduction_points = parsed_descriptor.introduction_points( + authentication_cookie=self.authentication_cookie + ) + + # If the new introduction points are different, flag this instance + # as modified. Compare the set of introduction point identifiers + # (fingerprint of the per IP circuit service key). + if (set(ip.identifier for ip in introduction_points) != + set(ip.identifier for ip in self.introduction_points)): + self.intro_set_changed_since_published = True + self.introduction_points = introduction_points + return True + + else: + logger.debug("Introduction points for instance %s.onion matched " + "the cached set.", self.onion_address) + return False + diff --git a/onionbalance/hs_v2/manager.py b/onionbalance/hs_v2/manager.py new file mode 100644 index 0000000..b56ee68 --- /dev/null +++ b/onionbalance/hs_v2/manager.py @@ -0,0 +1,98 @@ +# -*- coding: utf-8 -*- +""" +Load balance a hidden service across multiple (remote) Tor instances by +create a hidden service descriptor containing introduction points from +each instance. +""" +import os +import sys +import logging + +# import Crypto.PublicKey +import stem +from stem.control import Controller, EventType +from setproctitle import setproctitle # pylint: disable=no-name-in-module + +import onionbalance.common.util + +from onionbalance.common import scheduler +from onionbalance.common import log +from onionbalance.common import argparser +from onionbalance.common import signalhandler + +from onionbalance.hs_v2 import config +from onionbalance.hs_v2 import eventhandler +from onionbalance.hs_v2 import settings +from onionbalance.hs_v2 import status + +from onionbalance.hs_v2.service import publish_all_descriptors +from onionbalance.hs_v2.instance import fetch_instance_descriptors + +logger = log.get_logger() + +def main(): + """ + Entry point when invoked over the command line. + """ + setproctitle('onionbalance') + args = argparser.get_common_argparser().parse_args() + config_file_options = settings.parse_config_file(args.config) + + # Update global configuration with options specified in the config file + for setting in dir(config): + if setting.isupper() and config_file_options.get(setting): + setattr(config, setting, config_file_options.get(setting)) + + # Override the log level if specified on the command line. + if args.verbosity: + config.LOG_LEVEL = args.verbosity.upper() + + # Write log file if configured in environment variable or config file + if config.LOG_LOCATION: + log.setup_file_logger(config.LOG_LOCATION) + + logger.setLevel(logging.__dict__[config.LOG_LEVEL.upper()]) + + controller = onionbalance.common.util.connect_to_control_port(tor_socket, tor_address, tor_port, + config.TOR_CONTROL_PASSWORD) + + status_socket = status.StatusSocket(config.STATUS_SOCKET_LOCATION) + signalhandler.SignalHandler(controller, status_socket) + + # Disable no-member due to bug with "Instance of 'Enum' has no * member" + # pylint: disable=no-member + + # Check that the Tor client supports the HSPOST control port command + if not controller.get_version() >= stem.version.Requirement.HSPOST: + logger.error("A Tor version >= %s is required. You may need to " + "compile Tor from source or install a package from " + "the experimental Tor repository.", + stem.version.Requirement.HSPOST) + sys.exit(1) + + # Load the keys and config for each onion service + settings.initialize_services(controller, + config_file_options.get('services')) + + # Finished parsing all the config file. + + handler = eventhandler.EventHandler() + controller.add_event_listener(handler.new_status, + EventType.STATUS_CLIENT) + controller.add_event_listener(handler.new_desc, + EventType.HS_DESC) + controller.add_event_listener(handler.new_desc_content, + EventType.HS_DESC_CONTENT) + + # Schedule descriptor fetch and upload events + scheduler.add_job(config.REFRESH_INTERVAL, fetch_instance_descriptors, + controller) + scheduler.add_job(config.PUBLISH_CHECK_INTERVAL, publish_all_descriptors) + + # Run initial fetch of HS instance descriptors + scheduler.run_all(delay_seconds=config.INITIAL_DELAY) + + # Begin main loop to poll for HS descriptors + scheduler.run_forever() + + return 0 diff --git a/onionbalance/service.py b/onionbalance/hs_v2/service.py similarity index 93% rename from onionbalance/service.py rename to onionbalance/hs_v2/service.py index 560ab50..ff973f7 100644 --- a/onionbalance/service.py +++ b/onionbalance/hs_v2/service.py @@ -6,11 +6,16 @@ import Crypto.PublicKey.RSA import stem -from onionbalance import descriptor -from onionbalance import util -from onionbalance import log -from onionbalance import config -from onionbalance import consensus +from onionbalance.hs_v2 import descriptor +from onionbalance.hs_v2 import util +from onionbalance.hs_v2 import consensus + +import onionbalance.common.descriptor +import onionbalance.common.util + +from onionbalance.common import log +from onionbalance.hs_v2 import config +from onionbalance.common import intro_point_set logger = log.get_logger() @@ -63,7 +68,7 @@ def _intro_points_modified(self): Check if the introduction point set has changed since last publish. """ - return any(instance.changed_since_published + return any(instance.intro_set_changed_since_published for instance in self.instances) def _descriptor_not_uploaded_recently(self): @@ -97,7 +102,7 @@ def _select_introduction_points(self): """ Choose set of introduction points from all fresh descriptors - Returns a descriptor.IntroductionPointSet() which can be used to + Returns an intro_point_set.IntroductionPointSet() which can be used to choose introduction points. """ available_intro_points = [] @@ -137,10 +142,10 @@ def _select_introduction_points(self): continue else: # Include this instance's introduction points - instance.changed_since_published = False + instance.intro_set_changed_since_published = False available_intro_points.append(instance.introduction_points) - return descriptor.IntroductionPointSet(available_intro_points) + return intro_point_set.IntroductionPointSet(available_intro_points) def _publish_descriptor(self, deviation=0): """ @@ -237,7 +242,7 @@ def _upload_descriptor(self, signed_descriptor, replica, hsdirs=None): while True: try: - descriptor.upload_descriptor(self.controller, + onionbalance.common.descriptor.upload_descriptor(self.controller, signed_descriptor, hsdirs=hsdirs) break @@ -245,7 +250,7 @@ def _upload_descriptor(self, signed_descriptor, replica, hsdirs=None): logger.error("Error uploading descriptor for service " "%s.onion, Socket is closed.", self.onion_address) - util.reauthenticate(self.controller, logger) + onionbalance.common.util.reauthenticate(self.controller, logger) except stem.ControllerError: logger.exception("Error uploading descriptor for service " "%s.onion.", self.onion_address) diff --git a/onionbalance/hs_v2/settings.py b/onionbalance/hs_v2/settings.py new file mode 100644 index 0000000..eeeff66 --- /dev/null +++ b/onionbalance/hs_v2/settings.py @@ -0,0 +1,106 @@ +# -*- coding: utf-8 -*- + +""" +Implements the generation and loading of configuration files. +""" +from builtins import input, range +import os +import sys +import errno +import argparse +import logging + +import yaml + +from onionbalance.hs_v2 import config +from onionbalance.common import util +from onionbalance.common import log + +import onionbalance.hs_v2.util +import onionbalance.common.util + +import onionbalance.hs_v2.service +import onionbalance.hs_v2.instance + +logger = log.get_logger() + + +def parse_config_file(config_file): + """ + Parse config file containing service information + """ + config_path = os.path.abspath(config_file) + config_data = onionbalance.common.util.read_config_data_from_file(config_path) + + # Rewrite relative paths in the config to be relative to the config + # file directory + config_directory = os.path.dirname(config_path) + for service in config_data.get('services'): + if not os.path.isabs(service.get('key')): + service['key'] = os.path.join(config_directory, service['key']) + + return config_data + + +def initialize_services(controller, services_config): + """ + Load keys for services listed in the config + """ + + # Load the keys and config for each onion service + for service in services_config: + try: + service_key = onionbalance.hs_v2.util.key_decrypt_prompt(service.get("key")) + except (IOError, OSError) as e: + if e.errno == errno.ENOENT: + logger.error("Private key file %s could not be found. " + "Relative paths in the config file are loaded " + "relative to the config file directory.", + service.get("key")) + sys.exit(1) + elif e.errno == errno.EACCES: + logger.error("Permission denied to private key %s.", + service.get("key")) + sys.exit(1) + else: + raise + # Key file was read but a valid private key was not found. + if not service_key: + logger.error("Private key %s could not be loaded. It is a not " + "valid 1024 bit PEM encoded RSA private key", + service.get("key")) + sys.exit(1) + else: + # Successfully imported the private key + onion_address = onionbalance.hs_v2.util.calc_onion_address(service_key) + logger.debug("Loaded private key for service %s.onion.", + onion_address) + + # Load all instances for the current onion service + instance_config = service.get("instances", []) + if not instance_config: + logger.error("Could not load any instances for service " + "%s.onion.", onion_address) + sys.exit(1) + else: + instances = [] + for instance in instance_config: + instances.append(onionbalance.hs_v2.instance.InstanceV2( + controller=controller, + onion_address=instance.get("address"), + authentication_cookie=instance.get("auth") + )) + + logger.info("Loaded %d instances for service %s.onion.", + len(instances), onion_address) + + # Store service configuration in config.services global + config.services.append(onionbalance.hs_v2.service.Service( + controller=controller, + service_key=service_key, + instances=instances + )) + + # Store a global reference to current controller connection + config.controller = controller + diff --git a/onionbalance/status.py b/onionbalance/hs_v2/status.py similarity index 98% rename from onionbalance/status.py rename to onionbalance/hs_v2/status.py index f6de9ac..ec09a4a 100644 --- a/onionbalance/status.py +++ b/onionbalance/hs_v2/status.py @@ -9,8 +9,8 @@ import socket from socketserver import BaseRequestHandler, ThreadingMixIn, UnixStreamServer -from onionbalance import log -from onionbalance import config +from onionbalance.common import log +from onionbalance.hs_v2 import config logger = log.get_logger() diff --git a/onionbalance/util.py b/onionbalance/hs_v2/util.py similarity index 93% rename from onionbalance/util.py rename to onionbalance/hs_v2/util.py index 9d29a7f..3707749 100644 --- a/onionbalance/util.py +++ b/onionbalance/hs_v2/util.py @@ -6,13 +6,12 @@ import base64 import binascii import os -import stem import time # import Crypto.Util import Crypto.PublicKey -from onionbalance import config +from onionbalance.hs_v2 import config def add_pkcs1_padding(message): @@ -166,12 +165,3 @@ def is_directory_empty(path): return True -def reauthenticate(controller, logger): - """ - Tries to authenticate to the controller - """ - time.sleep(10) - try: - controller.authenticate(password=config.TOR_CONTROL_PASSWORD) - except stem.connection.AuthenticationFailure: - logger.error("Failed to re-authenticate controller.") diff --git a/onionbalance/hs_v3/__init__.py b/onionbalance/hs_v3/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/onionbalance/hs_v3/consensus.py b/onionbalance/hs_v3/consensus.py new file mode 100644 index 0000000..5ed1ff0 --- /dev/null +++ b/onionbalance/hs_v3/consensus.py @@ -0,0 +1,274 @@ +import datetime +import os +import base64 +import hashlib +import logging + +import stem +import stem.util +import stem.descriptor.remote +from stem.descriptor import DocumentHandler +from stem.descriptor import parse_file +from stem.descriptor.remote import DescriptorDownloader +from stem.descriptor.networkstatus import NetworkStatusDocumentV3 + +from onionbalance.common import log +from onionbalance.hs_v3 import tor_node + +logger = log.get_logger() + +class Consensus(object): + """ + This represents a consensus object. + + It's initialized once in startup and refreshed during the runtime using the + refresh() method to get the latest consensus. + """ + def __init__(self, do_refresh_consensus=True): + self.nodes = None + self.consensus = None + + if not do_refresh_consensus: + return + + # Set self.consensus to a NetworkStatusDocumentV3 object + # and initialize the nodelist + self.refresh() + + def refresh(self): + from onionbalance.hs_v3.onionbalance import my_onionbalance + + # Fetch the current md consensus from the control port + md_consensus_str = my_onionbalance.controller.get_md_consensus().encode() + try: + self.consensus = NetworkStatusDocumentV3(md_consensus_str) + except ValueError: + logger.warning("No valid consensus received. Waiting for one...") + return + + # Check if it's live + if not self.is_live(): + logger.info("Loaded consensus is not live. Waiting for a live one.") + return + + self.nodes = self._initialize_nodes() + + def get_routerstatuses(self): + """Give access to the routerstatus entries in this consensus""" + + # We should never be asked for routerstatuses with a non-live consensus + # so make sure this is the case. + assert(self.is_live()) + + return self.consensus.routers + + def is_live(self): + """ + Return True if the consensus is live. + """ + if not self.consensus: + return False + + now = datetime.datetime.utcnow() + return self.consensus.valid_after <= now and now <= self.consensus.valid_until + + def _initialize_nodes(self): + from onionbalance.hs_v3.onionbalance import my_onionbalance + + nodes = [] + + try: + microdescriptors_list = list(my_onionbalance.controller.controller.get_microdescriptors()) + except stem.DescriptorUnavailable: + logger.warning("Can't get microdescriptors from Tor. Delaying...") + return + + # Turn the mds into a dictionary indexed by the digest as an + # optimization while matching them with routerstatuses. + microdescriptors_dict = {} + for md in microdescriptors_list: + microdescriptors_dict[md.digest()] = md + + # Go through the routerstatuses and match them up with + # microdescriptors, and create a Node object for each match. If there + # is no match we don't register it as a node. + for relay_fpr, relay_routerstatus in self.get_routerstatuses().items(): + logger.debug("Checking routerstatus with md digest %s", relay_routerstatus.microdescriptor_digest) + + # Skip routerstatuses for which we cannot find a microdescriptor + if relay_routerstatus.microdescriptor_digest not in microdescriptors_dict: + logger.debug("Could not find microdesc for rs with fpr %s", relay_fpr) + continue + + node_microdescriptor = microdescriptors_dict[relay_routerstatus.microdescriptor_digest] + node = tor_node.Node(node_microdescriptor, relay_routerstatus) + nodes.append(node) + + logger.debug("Initialized %d nodes (%d routerstatuses / %d microdescriptors)", + len(nodes), len(self.get_routerstatuses()), len(microdescriptors_list)) + + return nodes + + def _get_disaster_srv(self, time_period_num): + """ + Return disaster SRV for 'time_period_num'. + """ + from onionbalance.hs_v3.onionbalance import my_onionbalance + time_period_length = self.get_time_period_length() + + disaster_body = b"shared-random-disaster" + time_period_length.to_bytes(8, 'big') + time_period_num.to_bytes(8, 'big') + return hashlib.sha3_256(disaster_body).digest() + + def get_current_srv(self, time_period_num): + if self.consensus.shared_randomness_current_value: + return base64.b64decode(self.consensus.shared_randomness_current_value) + elif time_period_num: + logger.info("SRV not found so falling back to disaster mode") + return self._get_disaster_srv(time_period_num) + else: + return None + + def get_previous_srv(self, time_period_num): + if self.consensus.shared_randomness_previous_value: + return base64.b64decode(self.consensus.shared_randomness_previous_value) + elif time_period_num: + logger.info("SRV not found so falling back to disaster mode") + return self._get_disaster_srv(time_period_num) + else: + return None + + def _get_srv_phase_duration(self): + """ + Return the length of the phase of a shared random protocol run in minutes. + """ + + # Each SRV phase takes 12 rounds. But the duration of the round depends + # on how big the voting rounds are which differs between live and + # testing network: + from onionbalance.hs_v3.onionbalance import my_onionbalance + if my_onionbalance.is_testnet: + return (12 * 20) // 60 + else: + return 12 * 60 + + def get_time_period_length(self): + """ + Get the HSv3 time period length in minutes + """ + from onionbalance.hs_v3.onionbalance import my_onionbalance + if my_onionbalance.is_testnet: + # This is a chutney network! Use hs_common.c:get_time_period_length() + # logic to calculate time period length + return (24 * 20) // 60 + else: + # This is not a chutney network, so time period length is 1440 minutes (24 hours) + return 24 * 60 + + def get_blinding_param(self, identity_pubkey, time_period_number): + """ + Calculate the HSv3 blinding parameter as specified in rend-spec-v3.txt section A.2: + + h = H(BLIND_STRING | A | s | B | N) + BLIND_STRING = "Derive temporary signing key" | INT_1(0) + N = "key-blind" | INT_8(period-number) | INT_8(period_length) + B = "(1511[...]2202, 4631[...]5960)" + + Use the time period number in 'time_period_number'. + """ + ED25519_BASEPOINT = b"(15112221349535400772501151409588531511" \ + b"454012693041857206046113283949847762202, " \ + b"463168356949264781694283940034751631413" \ + b"07993866256225615783033603165251855960)" + BLIND_STRING = b"Derive temporary signing key" + bytes([0]) + + period_length = self.get_time_period_length() + N = b"key-blind" + time_period_number.to_bytes(8, 'big') + period_length.to_bytes(8, 'big') + + return hashlib.sha3_256(BLIND_STRING + identity_pubkey + ED25519_BASEPOINT + N).digest() + + def get_next_time_period_num(self, valid_after=None): + return self.get_time_period_num(valid_after) + 1 + + def get_time_period_num(self, valid_after=None): + """ + Get time period number for this 'valid_after'. + + valid_after is a datetime (if not set, we get it ourselves) + time_period_length set to default value of 1440 minutes == 1 day + """ + if not valid_after: + assert(self.is_live()) + valid_after = self.consensus.valid_after + valid_after = stem.util.datetime_to_unix(valid_after) + + time_period_length = self.get_time_period_length() + + seconds_since_epoch = valid_after + minutes_since_epoch = seconds_since_epoch // 60 + + # Calculate offset as specified in rend-spec-v3.txt [TIME-PERIODS] + time_period_rotation_offset = self._get_srv_phase_duration() + + assert(minutes_since_epoch > time_period_rotation_offset) + minutes_since_epoch -= time_period_rotation_offset + + time_period_num = minutes_since_epoch // time_period_length + return int(time_period_num) + + def get_start_time_of_current_srv_run(self): + """ + Return the start time of the current SR protocol run using the times from + the current consensus. For example, if the latest consensus valid-after is + 23/06/2017 23:00:00 and a full SR protocol run is 24 hours, this function + returns 23/06/2017 00:00:00. + + TODO: unittest + """ + from onionbalance.hs_v3.onionbalance import my_onionbalance + + assert(self.is_live()) + + beginning_of_current_round = stem.util.datetime_to_unix(self.consensus.valid_after) + + # Voting interval is 20 secs in chutney and one hour in real network + if my_onionbalance.is_testnet: + voting_interval_secs = 20 + else: + voting_interval_secs = 60*60 + + # Get current SR protocol round (an SR protocol run has 24 rounds) + curr_round_slot = (beginning_of_current_round // voting_interval_secs) % 24 + time_elapsed_since_start_of_run = curr_round_slot * voting_interval_secs + + logger.debug("Current SRV proto run: Start of current round: %s. " + "Time elapsed: %s (%s)", beginning_of_current_round, + time_elapsed_since_start_of_run, voting_interval_secs) + + return int(beginning_of_current_round - time_elapsed_since_start_of_run) + + def get_start_time_of_previous_srv_run(self): + from onionbalance.hs_v3.onionbalance import my_onionbalance + + start_time_of_current_run = self.get_start_time_of_current_srv_run() + if my_onionbalance.is_testnet: + return start_time_of_current_run - 24*20 + else: + return start_time_of_current_run - 24*3600 + + def get_start_time_of_next_time_period(self, valid_after=None): + """ + Return the start time of the upcoming time period + """ + assert(self.is_live()) + + # Get start time of next time period + time_period_length = self.get_time_period_length() + next_time_period_num = self.get_next_time_period_num(valid_after) + start_of_next_tp_in_mins = next_time_period_num * time_period_length; + + # Apply rotation offset as specified by prop224 section [TIME-PERIODS] + time_period_rotation_offset = self._get_srv_phase_duration() + + return (start_of_next_tp_in_mins + time_period_rotation_offset)*60 + +class NoLiveConsensus(Exception): pass diff --git a/onionbalance/hs_v3/descriptor.py b/onionbalance/hs_v3/descriptor.py new file mode 100644 index 0000000..f9b9f30 --- /dev/null +++ b/onionbalance/hs_v3/descriptor.py @@ -0,0 +1,276 @@ +import logging +import datetime +import hashlib +import itertools +import binascii + +import stem.util +from stem.descriptor.hidden_service import HiddenServiceDescriptorV3, IntroductionPointV3, InnerLayer +from stem.descriptor.certificate import Ed25519CertificateV1, Ed25519Extension, ExtensionType + +from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from cryptography.hazmat.primitives.asymmetric.x25519 import X25519PrivateKey, X25519PublicKey +from cryptography.hazmat.primitives.ciphers import Cipher, algorithms, modes +from cryptography.hazmat.primitives.ciphers.algorithms import AES +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.backends import default_backend + +from onionbalance.common import log +from onionbalance.common import intro_point_set + +from onionbalance.hs_v3 import params + +logger = log.get_logger() +backend = default_backend() + +class IntroductionPointSetV3(intro_point_set.IntroductionPointSet): + """ + This class represents a set of introduction points (which are actually + stem.descriptor.hidden_service.IntroductionPointV3 objects) + + It gives us a nice way to compare sets of introduction poinst between them, + to see if they are different. + + It also preserves all the functionality of + onionbalance.common.intro_point_set.IntroductionPointSet which allows you to + sample introduction points out of the set. + """ + def __init__(self, introduction_points): + """ + 'introduction_points' is a list of lists where each internal list contains + the introduction points of an instance + """ + for instance_ips in introduction_points: + for ip in instance_ips: + if ip.legacy_key_raw: + logger.info("Ignoring introduction point with legacy key.") + instance_ips.remove(ip) + + super().__init__(introduction_points) + + def get_intro_points_flat(self): + """ + Flatten the .intro_points list of list into a single list and return it + """ + return list(itertools.chain(*self.intro_points)) + + def __eq__(self, other): + """ + Compares two IntroductionPointSetV3 objects and returns True + if they have the same introduction points in them. + """ + # XXX we are currently using onion_key_raw as the identifier for the + # intro point. is there a better thing to use? + intro_set_1 = set(ip.onion_key_raw for ip in other.get_intro_points_flat()) + intro_set_2 = set(ip.onion_key_raw for ip in self.get_intro_points_flat()) + + # TODO: unittests + return intro_set_1 == intro_set_2 + +class V3Descriptor(object): + """ + A generic v3 descriptor. + + Serves as the base class for OBDescriptor and ReceivedDescriptor which + implement more specific functionalities. + """ + def __init__(self, onion_address, v3_desc): + self.onion_address = onion_address + + self.v3_desc = v3_desc + + # An IntroductionPointSetV3 object with the intros of this descriptor + self.intro_set = IntroductionPointSetV3([self.v3_desc._inner_layer.introduction_points]) + + def get_intro_points(self): + """ + Get the raw intro points for this descriptor. + """ + return self.intro_set.get_intro_points_flat() + + def get_blinded_key(self): + """ + Extract and return the blinded key from the descriptor + """ + + # The descriptor signing cert, signs the descriptor signing key using + # the blinded key. So the signing key should be the one we want here. + return self.v3_desc.signing_cert.signing_key() + + def get_size(self): + """ + Return size of v3 descriptor in bytes + """ + return len(str(self.v3_desc)) + +class OBDescriptor(V3Descriptor): + """ + A v3 descriptor created by Onionbalance and meant to be published to the + network. + + This class supports generating descriptors. + + Can raise BadDescriptor if we can't or should not generate a valid descriptor. + """ + def __init__(self, onion_address, identity_priv_key, + blinding_param, intro_points, is_first_desc): + # Timestamp we last uploaded this descriptor + self.last_upload_ts = None + # Set of responsible HSDirs for last time we uploaded this descriptor + self.responsible_hsdirs = None + + # Start generating descriptor + desc_signing_key = Ed25519PrivateKey.generate() + + # Get the intro points for this descriptor and recertify them! + recertified_intro_points = [] + for ip in intro_points: + recertified_intro_points.append(self._recertify_intro_point(ip, desc_signing_key)) + + rev_counter = self._get_revision_counter(identity_priv_key, is_first_desc) + + v3_desc_inner_layer = InnerLayer.create(introduction_points = recertified_intro_points) + v3_desc = HiddenServiceDescriptorV3.create( + blinding_nonce = blinding_param, + identity_key = identity_priv_key, + signing_key = desc_signing_key, + inner_layer = v3_desc_inner_layer, + revision_counter = int(rev_counter), + ) + + # TODO stem should probably initialize it itself so that it has balance + # between descriptor creation (where this is not inted) and descriptor + # parsing (where this is inited) + v3_desc._inner_layer = v3_desc_inner_layer + + # Check max size is within range + if len(str(v3_desc)) > params.MAX_DESCRIPTOR_SIZE: + logger.error("Created descriptor is too big (%d intros). Consider "\ + "relaxing number of instances or intro points per instance "\ + "(see N_INTROS_PER_INSTANCE)") + raise BadDescriptor + + super().__init__(onion_address, v3_desc) + + def set_last_upload_ts(self, last_upload_ts): + self.last_upload_ts = last_upload_ts + + def set_responsible_hsdirs(self, responsible_hsdirs): + self.responsible_hsdirs = responsible_hsdirs + + def _recertify_intro_point(self, intro_point, descriptor_signing_key): + """ + Given an IntroductionPointV3 object, re-certify it using the + 'descriptor_signing_key' for this new descriptor. + + Return the recertified intro point. + """ + original_auth_key_cert = intro_point.auth_key_cert + original_enc_key_cert = intro_point.enc_key_cert + + # We have already removed all the intros with legacy keys. Make sure that + # no legacy intros sneaks up on us, becausey they would result in + # unparseable descriptors if we don't recertify them (and we won't). + assert(not intro_point.legacy_key_cert) + + # Get all the certs we need to recertify + # [we need to use the _replace method of namedtuples because there is no + # setter for those attributes due to the way stem sets those fields. If we + # attempt to normally replace the attributes we get the following + # exception: AttributeError: can't set attribute] + recertified_intro_point = intro_point._replace(auth_key_cert = self._recertify_ed_certificate(original_auth_key_cert, + descriptor_signing_key), + enc_key_cert = self._recertify_ed_certificate(original_enc_key_cert, + descriptor_signing_key)) + + + return recertified_intro_point + + def _recertify_ed_certificate(self, ed_cert, descriptor_signing_key): + """ + Recertify an HSv3 intro point certificate using the new descriptor signing + key so that it can be accepted as part of a new descriptor. + + "Recertifying" means taking the certified key and signing it with a new + key. + + Return the new certificate. + """ + extensions = [Ed25519Extension(ExtensionType.HAS_SIGNING_KEY, None, stem.util._pubkey_bytes(descriptor_signing_key))] + new_cert = Ed25519CertificateV1(cert_type = ed_cert.type, + expiration = ed_cert.expiration, + key_type = ed_cert.key_type, + key = ed_cert.key, + extensions = extensions, + signing_key=descriptor_signing_key) + + return new_cert + + + def _get_revision_counter(self, identity_priv_key, is_first_desc): + """ + Get the revision counter using the order-preserving-encryption scheme from + rend-spec-v3.txt section F.2. + """ + from onionbalance.hs_v3.onionbalance import my_onionbalance + now = int(stem.util.datetime_to_unix(datetime.datetime.utcnow())) + + # TODO: Mention that this is done with the private key instead of the blinded priv key + # this means that this won't cooperate with normal tor + privkey_bytes = identity_priv_key.private_bytes(encoding=serialization.Encoding.Raw, + format=serialization.PrivateFormat.Raw, + encryption_algorithm=serialization.NoEncryption()) + cipher_key = hashlib.sha3_256(b"rev-counter-generation" + privkey_bytes).digest() + + if is_first_desc: + srv_start = my_onionbalance.consensus.get_start_time_of_previous_srv_run() + else: + srv_start = my_onionbalance.consensus.get_start_time_of_current_srv_run() + srv_start = int(srv_start) + + seconds_since_srv_start = now - srv_start + # This must be strictly positive + seconds_since_srv_start += 1 + + ope_result = sum(w for w, _ in zip(self._get_ope_scheme_words(cipher_key), + range(seconds_since_srv_start))) + + logger.debug("Rev counter for %s descriptor (SRV secs %s, OPE %s)", + "first" if is_first_desc else "second", + seconds_since_srv_start, ope_result) + + return ope_result + + def _get_ope_scheme_words(self, cipher_key): + IV = b'\x00' * 16 + + cipher = Cipher(algorithms.AES(cipher_key), modes.CTR(IV), backend=backend) + e = cipher.encryptor() + while True: + v = e.update(b'\x00\x00') + yield v[0] + 256 * v[1] + 1 + +class ReceivedDescriptor(V3Descriptor): + """ + An instance v3 descriptor received from the network. + + This class supports parsing descriptors. + """ + def __init__(self, desc_text, onion_address): + """ + Parse a descriptor in 'desc_text' and return an ReceivedDescriptor object. + + Raises BadDescriptor if the descriptor cannot be used. + """ + try: + v3_desc = HiddenServiceDescriptorV3.from_str(desc_text) + v3_desc.decrypt(onion_address) + except ValueError as err: + logger.warning("Descriptor is corrupted (%s).", err) + raise BadDescriptor + + logger.debug("Successfuly decrypted descriptor for %s!", onion_address) + + super().__init__(onion_address, v3_desc) + +class BadDescriptor(Exception): pass diff --git a/onionbalance/hs_v3/hashring.py b/onionbalance/hs_v3/hashring.py new file mode 100644 index 0000000..6f30c33 --- /dev/null +++ b/onionbalance/hs_v3/hashring.py @@ -0,0 +1,228 @@ +import logging +import datetime +import base64 +import bisect +import hashlib + +from cryptography.hazmat.primitives import serialization + +import stem.util + +from onionbalance.common import log + +from onionbalance.hs_v3 import tor_node +from onionbalance.hs_v3 import params + +logger = log.get_logger() + +class HSV3HashRing(object): + def __init__(self): + self.hsdir_n_replicas = params.HSDIR_N_REPLICAS + self.hsdir_spread_store = params.HSDIR_SPREAD_STORE + + def _time_between_tp_and_srv(self, valid_after): + """ + Return True if we are currently in the time segment between a new time + period and a new SRV (in the real network that happens between 12:00 and + 00:00 UTC). Here is a diagram showing exactly when this returns true: + + +------------------------------------------------------------------+ + | | + | 00:00 12:00 00:00 12:00 00:00 12:00 | + | SRV#1 TP#1 SRV#2 TP#2 SRV#3 TP#3 | + | | + | $==========|-----------$===========|-----------$===========| | + | ^^^^^^^^^^^^ ^^^^^^^^^^^^ | + | | + +------------------------------------------------------------------+ + """ + from onionbalance.hs_v3.onionbalance import my_onionbalance + + srv_start_time = my_onionbalance.consensus.get_start_time_of_current_srv_run() + tp_start_time = my_onionbalance.consensus.get_start_time_of_next_time_period(srv_start_time) + valid_after = stem.util.datetime_to_unix(valid_after) + + if valid_after >= srv_start_time and valid_after < tp_start_time: + logger.debug("We are between SRV and TP") + return False + + logger.debug("We are between TP and SRV (valid_after: %s, srv_start_time: %s -> tp_start_time: %s)", + valid_after, srv_start_time, tp_start_time) + return True + + def _get_srv_and_time_period(self, is_first_descriptor): + """ + Return SRV and time period based on current consensus time + """ + from onionbalance.hs_v3.onionbalance import my_onionbalance + + valid_after = my_onionbalance.consensus.consensus.valid_after + + current_tp = my_onionbalance.consensus.get_time_period_num() + previous_tp = current_tp - 1 + next_tp = current_tp + 1 + assert(previous_tp > 0) + + # Get the right TP/SRV + if is_first_descriptor: + if self._time_between_tp_and_srv(valid_after): + srv = my_onionbalance.consensus.get_previous_srv(previous_tp) + tp = previous_tp + _case = 1 # just for debugging + else: + srv = my_onionbalance.consensus.get_previous_srv(current_tp) + tp = current_tp + _case = 2 # just for debugging + else: + if self._time_between_tp_and_srv(valid_after): + srv = my_onionbalance.consensus.get_current_srv(current_tp) + tp = current_tp + _case = 3 # just for debugging + else: + srv = my_onionbalance.consensus.get_current_srv(next_tp) + tp = next_tp + _case = 4 # just for debugging + + srv_b64 = base64.b64encode(srv) if srv else None + logger.debug("For valid_after %s we got SRV %s and TP %s (case: #%d)", + valid_after, srv_b64, tp, _case) + + return srv, tp + + + + + def _get_hash_ring_for_descriptor(self, is_first_descriptor): + """ + Return a dictionary { : Node , .... } + """ + from onionbalance.hs_v3.onionbalance import my_onionbalance + + node_hash_ring = {} + + srv, time_period_num = self._get_srv_and_time_period(is_first_descriptor) + logger.info("Using srv %s and TP#%s (%s descriptor)", + srv.hex(), time_period_num, + "first" if is_first_descriptor else "second") + + for node in my_onionbalance.consensus.nodes: + try: + hsdir_index = node.get_hsdir_index(srv, time_period_num) + except (tor_node.NoEd25519Identity, tor_node.NoHSDir) as e: + logger.debug("Could not find ed25519 for node %s", node.routerstatus.fingerprint) + continue + +# logger.debug("%s: Node: %s, index: %s", is_first_descriptor, node.get_hex_fingerprint(), hsdir_index.hex()) + node_hash_ring[hsdir_index] = node + + return node_hash_ring + + def _get_hidden_service_index(self, blinded_pubkey, replica_num, is_first_descriptor): + """ + hs_index(replicanum) = H("store-at-idx" | + blinded_public_key | + INT_8(replicanum) | + INT_8(period_length) | + INT_8(period_num) ) + """ + from onionbalance.hs_v3.onionbalance import my_onionbalance + period_length = my_onionbalance.consensus.get_time_period_length() + + replica_num_int_8 = replica_num.to_bytes(8, 'big') + period_length_int_8 = (period_length).to_bytes(8, 'big') + + _, time_period_num = self._get_srv_and_time_period(is_first_descriptor) + logger.info("Getting HS index with TP#%s for %s descriptor (%d replica) ", + time_period_num, + "first" if is_first_descriptor else "second", replica_num) + period_num_int_8 = time_period_num.to_bytes(8, 'big') + + hash_body = b"%s%s%s%s%s" % (b"store-at-idx", + blinded_pubkey, + replica_num_int_8, + period_length_int_8, + period_num_int_8) + + hs_index = hashlib.sha3_256(hash_body).digest() + + return hs_index + + + def get_responsible_hsdirs(self, blinded_pubkey, is_first_descriptor): + """ + Return a list with the responsible HSDirs for a service with 'blinded_pubkey'. + + The returned list is a list of fingerprints. + """ + from onionbalance.hs_v3.onionbalance import my_onionbalance + + # Always use a live consensus when calculating responsible HSDirs + assert(my_onionbalance.consensus.is_live()) + + responsible_hsdirs = [] + + # TODO: Improve representation of hash ring here... No need to go + # between list and dictionary... + + # dictionary { : Node , .... } + node_hash_ring = self._get_hash_ring_for_descriptor(is_first_descriptor) + if not node_hash_ring: + raise EmptyHashRing + + sorted_hash_ring_list = sorted(list(node_hash_ring.keys())) + + logger.info("Initialized hash ring of size %d (blinded key: %s)", + len(node_hash_ring), base64.b64encode(blinded_pubkey)) + + for replica_num in range(1, self.hsdir_n_replicas+1): + # The HSDirs that we are gonna store this replica in + replica_store_hsdirs = [] + + hidden_service_index = self._get_hidden_service_index(blinded_pubkey, replica_num, is_first_descriptor) + + # Find position of descriptor ID in the HSDir list + index = bisect.bisect_left(sorted_hash_ring_list, hidden_service_index) + + logger.info("\t Tried with HS index %s got position %d", hidden_service_index.hex(), index) + + while len(replica_store_hsdirs) < self.hsdir_spread_store: + try: + hsdir_key = sorted_hash_ring_list[index] + index += 1 + except IndexError: + # Wrap around when we reach the end of the HSDir list + index = 0 + hsdir_key = sorted_hash_ring_list[index] + + hsdir_node = node_hash_ring[hsdir_key] + + # Check if we have already added this node to this + # replica. This should never happen on the real network but + # might happen in small testnets like chutney! + if hsdir_node.get_hex_fingerprint() in replica_store_hsdirs: + logger.debug("Ignoring already added HSDir to this replica!") + break + + # Check if we have already added this node to the responsible + # HSDirs. This can happen in the second replica and we should + # skip the node + if hsdir_node.get_hex_fingerprint() in responsible_hsdirs: + logger.debug("Ignoring already added HSDir!") + continue + + logger.debug("%d: %s: %s", index, hsdir_node.get_hex_fingerprint(), hsdir_key.hex()) + + replica_store_hsdirs.append(hsdir_node.get_hex_fingerprint()) + + responsible_hsdirs.extend(replica_store_hsdirs) + + # Do a sanity check + if my_onionbalance.is_testnet: + # If we are on chutney it's normal to not have enough nodes to populate the hashring + assert(len(responsible_hsdirs) <= self.hsdir_n_replicas*self.hsdir_spread_store) + else: + assert(len(responsible_hsdirs) == self.hsdir_n_replicas*self.hsdir_spread_store) + + return responsible_hsdirs + +class EmptyHashRing(Exception): pass diff --git a/onionbalance/hs_v3/instance.py b/onionbalance/hs_v3/instance.py new file mode 100644 index 0000000..baacc61 --- /dev/null +++ b/onionbalance/hs_v3/instance.py @@ -0,0 +1,80 @@ +import datetime + +import onionbalance.common.instance +from onionbalance.common import log + +from onionbalance.hs_v3 import descriptor as ob_descriptor + +logger = log.get_logger() + +class InstanceV3(onionbalance.common.instance.Instance): + """ + This is a V3 onionbalance instance + """ + def __init__(self, onion_address): + # Get the controller + from onionbalance.hs_v3.onionbalance import my_onionbalance + controller = my_onionbalance.controller.controller + + # Initialize the common Instance class. + super().__init__(controller, onion_address) + + logger.warning("Loaded instance %s", onion_address) + + self.descriptor = None + + # When was the intro set of this instance last modified? + self.intro_set_modified_timestamp = None + + def register_descriptor(self, descriptor_text, onion_address): + """ + We received a descriptor (with 'descriptor_text') for 'onion_address'. + Register it to this instance. + """ + logger.info("Found instance %s for this new descriptor!", self.onion_address) + + assert(onion_address == self.onion_address) + + # Parse descriptor. If it parsed correctly, we know that this + # descriptor is truly for this instance (since the onion address + # matches) + try: + new_descriptor = ob_descriptor.ReceivedDescriptor(descriptor_text, onion_address) + except ob_descriptor.BadDescriptor: + logger.warning("Received bad descriptor for %s. Ignoring.", self.onion_address) + return + + # Before replacing the current descriptor with this one, check if the + # introduction point set changed: + + # If this is the first descriptor for this instance, the intro point set changed + if not self.descriptor: + logger.info("This is the first time we see a descriptor for instance %s!", self.onion_address) + self.intro_set_modified_timestamp = datetime.datetime.utcnow() + self.descriptor = new_descriptor + return + + assert(self.descriptor) + assert(new_descriptor.intro_set) + + # We already have a descriptor but this is a new one. Check the intro points! + if new_descriptor.intro_set != self.descriptor.intro_set: + logger.info("We got a new descriptor for instance %s and the intro set changed!", self.onion_address) + self.intro_set_modified_timestamp = datetime.datetime.utcnow() + else: + logger.info("We got a new descriptor for instance %s but the intro set did not change.", self.onion_address) + + self.descriptor = new_descriptor + + def get_intros_for_publish(self): + """ + Get a list of stem.descriptor.IntroductionPointV3 objects for this descriptor + + Raise :InstanceHasNoDescriptor: if there is no descriptor for this instance + """ + if not self.descriptor: + raise InstanceHasNoDescriptor + + return self.descriptor.get_intro_points() + +class InstanceHasNoDescriptor(Exception): pass diff --git a/onionbalance/hs_v3/manager.py b/onionbalance/hs_v3/manager.py new file mode 100644 index 0000000..e035d4a --- /dev/null +++ b/onionbalance/hs_v3/manager.py @@ -0,0 +1,57 @@ +from setproctitle import setproctitle # pylint: disable=no-name-in-module + +import logging + +from onionbalance import __version__ + +from onionbalance.common import scheduler +from onionbalance.common import log +from onionbalance.common import argparser +from onionbalance.common import signalhandler + +from onionbalance.hs_v3 import params +from onionbalance.hs_v3 import onionbalance + +logger = log.get_logger() + +def main(): + logger.warning("Initializing onionbalance (version: %s)...", __version__) + + setproctitle('onionbalance') + parser = argparser.get_common_argparser() + + parser.add_argument("--is-testnet", action='store_true', + help="Is this onionbalance on a test net? (Default: no)") + + args = parser.parse_args() + + # Override the log level if specified on the command line. + if args.verbosity: + params.DEFAULT_LOG_LEVEL = args.verbosity.upper() + logger.setLevel(logging.__dict__[params.DEFAULT_LOG_LEVEL.upper()]) + + # Get the global onionbalance singleton + my_onionbalance = onionbalance.my_onionbalance + my_onionbalance.init_subsystems(args) + + signalhandler.SignalHandler(my_onionbalance.controller.controller) + + # Schedule descriptor fetch and upload events + if my_onionbalance.is_testnet: + scheduler.add_job(params.FETCH_DESCRIPTOR_FREQUENCY_TESTNET, my_onionbalance.fetch_instance_descriptors) + scheduler.add_job(params.PUBLISH_DESCRIPTOR_CHECK_FREQUENCY_TESTNET, my_onionbalance.publish_all_descriptors) + else: + scheduler.add_job(params.FETCH_DESCRIPTOR_FREQUENCY, my_onionbalance.fetch_instance_descriptors) + scheduler.add_job(params.PUBLISH_DESCRIPTOR_CHECK_FREQUENCY, my_onionbalance.publish_all_descriptors) + + # Run initial fetch of HS instance descriptors + scheduler.run_all(delay_seconds=params.INITIAL_CALLBACK_DELAY) + + # Begin main loop to poll for HS descriptors + scheduler.run_forever() + + return 0 + +if __name__ == '__main__': + main() + diff --git a/onionbalance/hs_v3/onionbalance.py b/onionbalance/hs_v3/onionbalance.py new file mode 100644 index 0000000..2f2c068 --- /dev/null +++ b/onionbalance/hs_v3/onionbalance.py @@ -0,0 +1,211 @@ +import os, sys + +from stem.descriptor.hidden_service import HiddenServiceDescriptorV3 + +import onionbalance.common.instance + +from onionbalance.common import log + +from onionbalance.common import util + +from onionbalance.hs_v3 import stem_controller +from onionbalance.hs_v3 import hashring +from onionbalance.hs_v3 import service as ob_service +from onionbalance.hs_v3 import consensus as ob_consensus + +logger = log.get_logger() + +class Onionbalance(object): + """ + Onionbalance singleton that represents this onionbalance runtime + """ + def __init__(self): + # This is kept minimal so that it's quick (it's executed at program + # launch because of the onionbalance singleton). The actual init work + # happens in init_subsystems() + + # True if this onionbalance operates in a testnet (e.g. chutney) + self.is_testnet = False + + def init_subsystems(self, args): + """ + Initialize subsystems (this is resource intensive) + """ + self.args = args + self.config_path = os.path.abspath(self.args.config) + self.config_data = self.load_config_file() + self.is_testnet = args.is_testnet + + if self.is_testnet: + logger.warning("Onionbalance configured on a testnet!") + + # Create stem controller and connect to the Tor network + self.controller = stem_controller.StemController(args.ip, args.port) + self.consensus = ob_consensus.Consensus() + self.hash_ring = hashring.HSV3HashRing() + + # Initialize our service + self.services = self.initialize_services_from_config_data() + + # Catch interesting events (like receiving descriptors etc.) + self.controller.add_event_listeners() + + logger.warning("Onionbalance initialized!") + logger.warning("="*80) + + def initialize_services_from_config_data(self): + services = [] + for service in self.config_data['services']: + services.append(ob_service.OnionBalanceService(service, self.config_path)) + + if len(services) > 1: + # We don't know how to handle more than a single service right now + raise NotImplementedError + + return services + + def load_config_file(self): + config_data = util.read_config_data_from_file(self.config_path) + logger.debug("Onionbalance config data: %s", config_data) + + # Do some basic validation + if not "services" in config_data: + logger.error("Config file is bad. 'services' is missing. Did you make it with onionbalance-config?") + sys.exit(1) + + # More validation + for service in config_data["services"]: + if not "key" in service: + logger.error("Config file is bad. 'key' is missing. Did you make it with onionbalance-config?") + sys.exit(1) + + if not "instances" in service: + logger.error("Config file is bad. 'instances' is missing. Did you make it with onionbalance-config?") + sys.exit(1) + + for instance in service["instances"]: + if "address" not in instance: + logger.error("Config file is wrong. 'address' missing from instance.") + sys.exit(1) + + # Validate that the onion address is legit + try: + _ = HiddenServiceDescriptorV3.identity_key_from_address(instance["address"]) + except ValueError: + logger.error("Cannot load instance with bad address: '%s'", instance["address"]) + sys.exit(1) + + return config_data + + def fetch_instance_descriptors(self): + logger.info("[*] fetch_instance_descriptors() called [*]") + + # TODO: Don't do this here. Instead do it on a specialized function + self.controller.mark_tor_as_active() + + if not self.consensus.is_live(): + logger.warning("No live consensus. Waiting before fetching descriptors...") + return + + all_instances = self._get_all_instances() + + onionbalance.common.instance.helper_fetch_all_instance_descriptors(self.controller.controller, + all_instances) + + def handle_new_desc_content_event(self, desc_content_event): + """ + Parse HS_DESC_CONTENT response events for descriptor content + + Update the HS instance object with the data from the new descriptor. + """ + onion_address = desc_content_event.address + logger.debug("Received descriptor for %s.onion from %s", + onion_address, desc_content_event.directory) + + # Check that the HSDir returned a descriptor that is not empty + descriptor_text = str(desc_content_event.descriptor).encode('utf-8') + + # HSDirs provide a HS_DESC_CONTENT response with either one or two + # CRLF lines when they do not have a matching descriptor. Using + # len() < 5 should ensure all empty HS_DESC_CONTENT events are matched. + if len(descriptor_text) < 5: + logger.debug("Empty descriptor received for %s.onion", onion_address) + return None + + # OK this descriptor seems plausible: Find the instances that this + # descriptor belongs to: + for instance in self._get_all_instances(): + if instance.onion_address == onion_address: + instance.register_descriptor(descriptor_text, onion_address) + + def publish_all_descriptors(self): + """ + For each service attempt to publish all descriptors + """ + logger.info("[*] publish_all_descriptors() called [*]") + + if not self.consensus.is_live(): + logger.info("No live consensus. Waiting before publishing descriptors...") + return + + for service in self.services: + service.publish_descriptors() + + def _get_all_instances(self): + """ + Get all instances for all services + """ + instances = [] + + for service in self.services: + instances.extend(service.instances) + + return instances + + def handle_new_status_event(self, status_event): + """ + Parse Tor status events such as "STATUS_GENERAL" + """ + # pylint: disable=no-member + if status_event.action == "CONSENSUS_ARRIVED": + logger.info("Received new consensus!") + self.consensus.refresh() + # Call all callbacks in case we just got a live consensus + my_onionbalance.publish_all_descriptors() + my_onionbalance.fetch_instance_descriptors() + + def _address_is_instance(self, onion_address): + for service in self.services: + for instance in service.instances: + if instance.onion_address == onion_address: + return True + + def _address_is_frontend(self, onion_address): + for service in self.services: + if service.onion_address == onion_address: + return True + + def handle_new_desc_event(self, desc_event): + """ + Parse HS_DESC response events + """ + action = desc_event.action + + if action == "RECEIVED": + pass # We already log in HS_DESC_CONTENT so no need to do it here too + elif action == "UPLOADED": + logger.debug("Successfully uploaded descriptor for %s to %s", desc_event.address, desc_event.directory) + elif action == "FAILED": + if self._address_is_instance(desc_event.address): + logger.info("Description fetch failed for instance %s to %s", desc_event.address, desc_event.directory) + elif self._address_is_frontend(desc_event.address): + logger.warning("Descriptor upload failed for frontend %s to %s", desc_event.address, desc_event.directory) + else: + logger.warning("Descriptor action failed for unknown service %s to %s", desc_event.address, desc_event.directory) + elif action == "REQUESTED": + logger.debug("Requested descriptor for %s from %s...", desc_event.address, desc_event.directory) + else: + pass + + +my_onionbalance = Onionbalance() diff --git a/onionbalance/hs_v3/params.py b/onionbalance/hs_v3/params.py new file mode 100644 index 0000000..a3b46bc --- /dev/null +++ b/onionbalance/hs_v3/params.py @@ -0,0 +1,45 @@ +import os + +#### Parameters definining Onionbalance behavior + +# How long to wait for onionbalance to bootstrap before starting periodic +# events (in seconds) +INITIAL_CALLBACK_DELAY = 45 + +# Every how often we should be fetching instance descriptors (in seconds) +FETCH_DESCRIPTOR_FREQUENCY = 10 * 60 +FETCH_DESCRIPTOR_FREQUENCY_TESTNET = 20 + +# Every how often we should be checking whether we should publish our frontend +# descriptor (in seconds). Triggering this callback doesn't mean we will +# actually upload a descriptor. We only upload a descriptor if it has expired, +# the intro points have changed, etc. +PUBLISH_DESCRIPTOR_CHECK_FREQUENCY = 5 * 60 +PUBLISH_DESCRIPTOR_CHECK_FREQUENCY_TESTNET = 10 + +# How long should we keep a frontend descriptor before we expire it (in +# seconds)? +FRONTEND_DESCRIPTOR_LIFETIME = 60 * 60 +FRONTEND_DESCRIPTOR_LIFETIME_TESTNET = 20 + +# How many intros should we use from each instance in the final frontend +# descriptor? +# [TODO: This makes no attempt to hide the use of onionbalance. In the future we +# should be smarter and sneakier here.] +N_INTROS_PER_INSTANCE = 2 + +#### Parameters defined by HSv3 spec and little-t-tor implementation + +# Number of replicas per descriptor +HSDIR_N_REPLICAS = 2 +# How many uploads per replica +# [TODO: Get these from the consensus instead of hardcoded] +HSDIR_SPREAD_STORE = 4 + +# Max descriptor size (in bytes) (see hs_cache_get_max_descriptor_size() in +# little-t-tor) +MAX_DESCRIPTOR_SIZE = 50000 + +#### Misc parameters + +DEFAULT_LOG_LEVEL = os.environ.get('ONIONBALANCE_LOG_LEVEL', 'warning') diff --git a/onionbalance/hs_v3/service.py b/onionbalance/hs_v3/service.py new file mode 100644 index 0000000..6d24571 --- /dev/null +++ b/onionbalance/hs_v3/service.py @@ -0,0 +1,331 @@ +import base64 +import datetime +import hashlib +import os +import logging + +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.backends import default_backend + +import stem +import stem.descriptor.hidden_service +from stem.descriptor.hidden_service import HiddenServiceDescriptorV3 + +import onionbalance.common.descriptor +from onionbalance.common import log +import onionbalance.common.util + +from onionbalance.hs_v3 import params +from onionbalance.hs_v3 import instance +from onionbalance.hs_v3 import hashring +from onionbalance.hs_v3 import descriptor + +logger = log.get_logger() + +class OnionBalanceService(object): + """ + Service represents a front-facing hidden service which should + be load-balanced. + """ + def __init__(self, service_config_data, config_path): + """ + With 'config_data' straight out of the config file, create the service and its instances. + 'config_path' is the full path to the config file. + + Raise ValueError if the config file is not well formatted + """ + # Load private key from config + self.identity_priv_key, self.onion_address = self._load_service_keys(service_config_data, config_path) + + # Now load up the instances + self.instances = self._load_instances(service_config_data) + + # First descriptor for this service (the one we uploaded last) + self.first_descriptor = None + # Second descriptor for this service (the one we uploaded last) + self.second_descriptor = None + + def _load_instances(self, service_config_data): + instances = [] + + for config_instance in service_config_data['instances']: + new_instance = instance.InstanceV3(config_instance['address']) + instances.append(new_instance) + + return instances + + def _load_service_keys(self, service_config_data, config_path): + # First of all let's load up the private key + key_fname = service_config_data['key'] + config_directory = os.path.dirname(config_path) + if not os.path.isabs(key_fname): + key_fname = os.path.join(config_directory, key_fname) + + with open(key_fname, 'rb') as handle: + pem_key_bytes = handle.read() + + identity_priv_key = serialization.load_pem_private_key(pem_key_bytes, password=None, backend=default_backend()) + + # Get onion address + identity_pub_key = identity_priv_key.public_key() + identity_pub_key_bytes = identity_pub_key.public_bytes(encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw) + onion_address = HiddenServiceDescriptorV3.address_from_identity_key(identity_pub_key_bytes) + + logger.warning("Loaded onion %s from %s", onion_address, key_fname) + + return identity_priv_key, onion_address + + def _intro_set_modified(self, is_first_desc): + """ + Check if the introduction point set has changed since last + publish. + """ + if is_first_desc: + last_upload_ts = self.first_descriptor.last_upload_ts + else: + last_upload_ts = self.second_descriptor.last_upload_ts + + if not last_upload_ts: + logger.info("\t Descriptor never published before. Do it now!") + return True + + for instance in self.instances: + if instance.intro_set_modified_timestamp: + logger.info("\t Still dont have a descriptor for this instance") + continue + + if instance.intro_set_modified_timestamp > last_upload_ts: + logger.info("\t Intro set modified") + return True + + logger.info("\t Intro set not modified") + return False + + def _get_descriptor_lifetime(self): + from onionbalance.hs_v3.onionbalance import my_onionbalance + if my_onionbalance.is_testnet: + return params.FRONTEND_DESCRIPTOR_LIFETIME_TESTNET + else: + return params.FRONTEND_DESCRIPTOR_LIFETIME + + def _descriptor_has_expired(self, is_first_desc): + """ + Check if the descriptor has expired (hasn't been uploaded recently). + + If 'is_first_desc' is set then check the first descriptor of the + service, otherwise the second. + """ + if is_first_desc: + last_upload_ts = self.first_descriptor.last_upload_ts + else: + last_upload_ts = self.second_descriptor.last_upload_ts + + descriptor_age = (datetime.datetime.utcnow() - last_upload_ts) + descriptor_age = int(descriptor_age.total_seconds()) + if (descriptor_age > self._get_descriptor_lifetime()): + logger.info("\t Our %s descriptor has expired (%s seconds old). Uploading new one.", + "first" if is_first_desc else "second", descriptor_age) + return True + else: + logger.info("\t Our %s descriptor is still fresh (%s seconds old).", + "first" if is_first_desc else "second", descriptor_age) + return False + + def _hsdir_set_changed(self, is_first_desc): + """ + Return True if the HSDir has changed between the last upload of this + descriptor and the current state of things + """ + from onionbalance.hs_v3.onionbalance import my_onionbalance + + # Derive blinding parameter + _, time_period_number = my_onionbalance.hash_ring._get_srv_and_time_period(is_first_desc) + blinded_param = my_onionbalance.consensus.get_blinding_param(self._get_identity_pubkey_bytes(), + time_period_number) + + # Get blinded key + # TODO: hoho! this is dirty we are poking into internal stem API. We + # should ask atagar to make it public for us! :) + blinded_key = stem.descriptor.hidden_service._blinded_pubkey(self._get_identity_pubkey_bytes(), blinded_param) + + # Calculate current responsible HSDirs + try: + responsible_hsdirs = my_onionbalance.hash_ring.get_responsible_hsdirs(blinded_key, is_first_desc) + except hashring.EmptyHashRing: + return False + + if is_first_desc: + previous_responsible_hsdirs = self.first_descriptor.responsible_hsdirs + else: + previous_responsible_hsdirs = self.second_descriptor.responsible_hsdirs + + if set(responsible_hsdirs) != set(previous_responsible_hsdirs): + logger.info("\t HSDir set changed (%s vs %s)", + set(responsible_hsdirs), set(previous_responsible_hsdirs)) + return True + else: + logger.info("\t HSDir set remained the same") + return False + + def _should_publish_descriptor_now(self, is_first_desc, force_publish=False): + """ + Return True if we should publish a descriptor right now + """ + # If descriptor not yet uploaded, do it now! + if is_first_desc and not self.first_descriptor: + return True + if not is_first_desc and not self.second_descriptor: + return True + + # OK this is not the first time we publish a descriptor. Check various + # parameters to see if we should try to publish again: + return any([self._intro_set_modified(is_first_desc), + self._descriptor_has_expired(is_first_desc), + self._hsdir_set_changed(is_first_desc), + force_publish]) + + def get_all_intros_for_publish(self): + """ + Return an IntroductionPointSetV3 with all the intros of all the instances + of this service. + """ + all_intros = [] + + for instance in self.instances: + try: + instance_intros = instance.get_intros_for_publish() + except onionbalance.hs_v3.instance.InstanceHasNoDescriptor: + logger.warning("Entirely missing a descriptor for instance %s. Continuing anyway if possible", + instance.onion_address) + continue + + all_intros.append(instance_intros) + + return descriptor.IntroductionPointSetV3(all_intros) + + def publish_descriptors(self): + self._publish_descriptor(is_first_desc=True) + self._publish_descriptor(is_first_desc=False) + + def _get_intros_for_desc(self): + all_intros = self.get_all_intros_for_publish() + + # Get number of instances that contributed to final intro point list + n_instances = len(all_intros.intro_points) + n_intros_wanted = n_instances * params.N_INTROS_PER_INSTANCE + + final_intros = all_intros.choose(n_intros_wanted) + + if (len(final_intros) == 0): + logger.info("Got no usable intro points from our instances. Delaying descriptor push...") + raise NotEnoughIntros + + logger.info("We got %d intros from %d instances. We want %d intros ourselves (got: %d)", + len(all_intros.get_intro_points_flat()), n_instances, + n_intros_wanted, len(final_intros)) + + return final_intros + + def _publish_descriptor(self, is_first_desc): + """ + Attempt to publish descriptor if needed. + + If 'is_first_desc' is set then attempt to upload the first descriptor + of the service, otherwise the second. + """ + from onionbalance.hs_v3.onionbalance import my_onionbalance + + if not self._should_publish_descriptor_now(is_first_desc): + logger.info("No reason to publish %s descriptor for %s", + "first" if is_first_desc else "second", + self.onion_address) + return + + try: + intro_points = self._get_intros_for_desc() + except NotEnoughIntros: + return + + # Derive blinding parameter + _, time_period_number = my_onionbalance.hash_ring._get_srv_and_time_period(is_first_desc) + blinding_param = my_onionbalance.consensus.get_blinding_param(self._get_identity_pubkey_bytes(), + time_period_number) + + try: + desc = descriptor.OBDescriptor(self.onion_address, self.identity_priv_key, + blinding_param, intro_points, is_first_desc) + except descriptor.BadDescriptor: + return + + logger.info("Service %s created %s descriptor (%s intro points) (blinding param: %s) (size: %s bytes). About to publish:", + self.onion_address, "first" if is_first_desc else "second", + len(desc.intro_set), blinding_param.hex(), len(str(desc.v3_desc))) + + # When we do a v3 HSPOST on the control port, Tor decodes the + # descriptor and extracts the blinded pubkey to be used when uploading + # the descriptor. So let's do the same to compute the responsible + # HSDirs: + blinded_key = desc.get_blinded_key() + + # Calculate responsible HSDirs for our service + try: + responsible_hsdirs = my_onionbalance.hash_ring.get_responsible_hsdirs(blinded_key, is_first_desc) + except hashring.EmptyHashRing: + logger.warning("Can't publish desc with no hash ring. Delaying...") + return + + logger.info("Uploading %s descriptor for %s to %s", + "first" if is_first_desc else "second", + self.onion_address, responsible_hsdirs) + + # Upload descriptor + self._upload_descriptor(my_onionbalance.controller.controller, + desc, responsible_hsdirs) + + # It would be better to set last_upload_ts when an upload succeeds and + # not when an upload is just attempted. Unfortunately the HS_DESC # + # UPLOADED event does not provide information about the service and + # so it can't be used to determine when descriptor upload succeeds + desc.set_last_upload_ts(datetime.datetime.utcnow()) + desc.set_responsible_hsdirs(responsible_hsdirs) + + # Set the descriptor + if is_first_desc: + self.first_descriptor = desc + else: + self.second_descriptor = desc + + def _upload_descriptor(self, controller, ob_desc, hsdirs): + """ + Convenience method to upload a descriptor + Handle some error checking and logging inside the Service class + """ + if hsdirs and not isinstance(hsdirs, list): + hsdirs = [hsdirs] + + while True: + try: + onionbalance.common.descriptor.upload_descriptor(controller, + ob_desc.v3_desc, + hsdirs=hsdirs, + v3_onion_address=ob_desc.onion_address) + break + except stem.SocketClosed: + logger.error("Error uploading descriptor for service " + "%s.onion, Socket is closed.", + self.onion_address) + onionbalance.common.util.reauthenticate(controller, logger) + except stem.ControllerError: + logger.exception("Error uploading descriptor for service " + "%s.onion.", self.onion_address) + break + + + + def _get_identity_pubkey_bytes(self): + identity_pub_key = self.identity_priv_key.public_key() + return identity_pub_key.public_bytes(encoding=serialization.Encoding.Raw, + format=serialization.PublicFormat.Raw) + +class NotEnoughIntros(Exception): pass diff --git a/onionbalance/hs_v3/stem_controller.py b/onionbalance/hs_v3/stem_controller.py new file mode 100644 index 0000000..3965441 --- /dev/null +++ b/onionbalance/hs_v3/stem_controller.py @@ -0,0 +1,64 @@ +import time +import traceback + +from stem.control import EventType, Signal +from stem import Signal + +import onionbalance.common.util +from onionbalance.common import log + +logger = log.get_logger() + +def handle_new_status_event_wrapper(status_event): + """ + A wrapper for this control port event. We need this so that we print + tracebacks on the listener thread (also see + https://stem.torproject.org/tutorials/tortoise_and_the_hare.html#advanced-listeners) + """ + from onionbalance.hs_v3.onionbalance import my_onionbalance + try: + my_onionbalance.handle_new_status_event(status_event) + except: + print(traceback.format_exc()) + +def handle_new_desc_event_wrapper(desc_event): + """ A wrapper for this control port event (see above) """ + from onionbalance.hs_v3.onionbalance import my_onionbalance + try: + my_onionbalance.handle_new_desc_event(desc_event) + except: + print(traceback.format_exc()) + +def handle_new_desc_content_event_wrapper(desc_content_event): + """ A wrapper for this control port event (see above) """ + from onionbalance.hs_v3.onionbalance import my_onionbalance + try: + my_onionbalance.handle_new_desc_content_event(desc_content_event) + except: + print(traceback.format_exc()) + +class StemController(object): + """This class is our interface to the control port""" + def __init__(self, address, port): + self.controller = onionbalance.common.util.connect_to_control_port(tor_address=address, + tor_port=port) + assert(self.controller.is_authenticated()) + + def mark_tor_as_active(self): + """ + Send the ACTIVE signal to the control port so that Tor does not become dormant. + """ + self.controller.signal(Signal.ACTIVE) + + def get_md_consensus(self): + return self.controller.get_info("dir/status-vote/current/consensus-microdesc") + + def add_event_listeners(self): + from onionbalance.hs_v3.onionbalance import my_onionbalance + self.controller.add_event_listener(handle_new_status_event_wrapper, EventType.STATUS_CLIENT) + self.controller.add_event_listener(handle_new_desc_event_wrapper, EventType.HS_DESC) + self.controller.add_event_listener(handle_new_desc_content_event_wrapper, EventType.HS_DESC_CONTENT) + + def shutdown(self): + self.controller.close() + diff --git a/onionbalance/hs_v3/tor_node.py b/onionbalance/hs_v3/tor_node.py new file mode 100644 index 0000000..cb51f75 --- /dev/null +++ b/onionbalance/hs_v3/tor_node.py @@ -0,0 +1,75 @@ +import logging +import base64 + +import hashlib + +from onionbalance.common import log + +logger = log.get_logger() + +class Node(object): + """ + Represents a Tor node. + + A Node instance gets created for each node of a consensus. When we fetch a + new consensus, we create new Node instances for the routers found inside. + + The 'microdescriptor' and 'routerstatus' fields of this object are + immutable: They are set once when we receive the consensus based on the + state of the network at that point, and they stay like that until we get a + new consensus. + """ + def __init__(self, microdescriptor, routerstatus): + assert(microdescriptor and routerstatus) + +# logger.debug("Initializing node with fpr %s", routerstatus.fingerprint) + + # if they are immutable why not also slap the consensus in here? + self.microdescriptor = microdescriptor + self.routerstatus = routerstatus + + def get_hex_fingerprint(self): + return self.routerstatus.fingerprint + + def get_hsdir_index(self, srv, period_num): + """ + hsdir_index(node) = H("node-idx" | node_identity | + shared_random_value | + INT_8(period_num) | + INT_8(period_length) ) + """ + from onionbalance.hs_v3.onionbalance import my_onionbalance + + # See if this node can be an HSDir (it needs to be supported both in + # protover and in flags) + if 'HSDir' not in self.routerstatus.protocols or \ + 2 not in self.routerstatus.protocols['HSDir'] or \ + 'HSDir' not in self.routerstatus.flags: + raise NoHSDir + + # See if ed25519 identity is supported for this node + if 'ed25519' not in self.microdescriptor.identifiers: + raise NoEd25519Identity + + # In stem the ed25519 identity is a base64 string and we need to add + # the missing padding so that the python base64 module can successfuly + # decode it. + ed25519_node_identity_b64 = self.microdescriptor.identifiers['ed25519'] + missing_padding = len(ed25519_node_identity_b64) % 4 + ed25519_node_identity_b64 += '=' * missing_padding + ed25519_node_identity = base64.b64decode(ed25519_node_identity_b64) + + period_num_int_8 = period_num.to_bytes(8, 'big') + period_length = my_onionbalance.consensus.get_time_period_length() + period_length_int_8 = period_length.to_bytes(8, 'big') + + hash_body = b"%s%s%s%s%s" % (b"node-idx", + ed25519_node_identity, + srv, + period_num_int_8, period_length_int_8) + hsdir_index = hashlib.sha3_256(hash_body).digest() + + return hsdir_index + +class NoEd25519Identity(Exception): pass +class NoHSDir(Exception): pass diff --git a/onionbalance/hs_v3/util.py b/onionbalance/hs_v3/util.py new file mode 100644 index 0000000..f931f54 --- /dev/null +++ b/onionbalance/hs_v3/util.py @@ -0,0 +1,3 @@ +import logging +import stem + diff --git a/onionbalance/manager.py b/onionbalance/manager.py deleted file mode 100644 index d083be4..0000000 --- a/onionbalance/manager.py +++ /dev/null @@ -1,157 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Load balance a hidden service across multiple (remote) Tor instances by -create a hidden service descriptor containing introduction points from -each instance. -""" -import os -import sys -import argparse -import logging - -# import Crypto.PublicKey -import stem -from stem.control import Controller, EventType -from setproctitle import setproctitle # pylint: disable=no-name-in-module - -import onionbalance -from onionbalance import log -from onionbalance import settings -from onionbalance import config -from onionbalance import eventhandler -from onionbalance import status -from onionbalance import scheduler - -from onionbalance.service import publish_all_descriptors -from onionbalance.instance import fetch_instance_descriptors - -logger = log.get_logger() - - -def parse_cmd_args(): - """ - Parses and returns command line arguments. - """ - - parser = argparse.ArgumentParser( - description="onionbalance distributes the requests for a Tor hidden " - "services across multiple Tor instances.") - - parser.add_argument("-i", "--ip", type=str, default=None, - help="Tor controller IP address") - - parser.add_argument("-p", "--port", type=int, default=None, - help="Tor controller port") - - parser.add_argument("-s", "--socket", type=str, default=None, - help="Tor unix domain control socket location") - - parser.add_argument("-c", "--config", type=str, - default=os.environ.get('ONIONBALANCE_CONFIG', - "config.yaml"), - help="Config file location") - - parser.add_argument("-v", "--verbosity", type=str, default=None, - help="Minimum verbosity level for logging. Available " - "in ascending order: debug, info, warning, " - "error, critical). The default is info.") - - parser.add_argument('--version', action='version', - version='onionbalance %s' % onionbalance.__version__) - - return parser - - -def main(): - """ - Entry point when invoked over the command line. - """ - setproctitle('onionbalance') - args = parse_cmd_args().parse_args() - config_file_options = settings.parse_config_file(args.config) - - # Update global configuration with options specified in the config file - for setting in dir(config): - if setting.isupper() and config_file_options.get(setting): - setattr(config, setting, config_file_options.get(setting)) - - # Override the log level if specified on the command line. - if args.verbosity: - config.LOG_LEVEL = args.verbosity.upper() - - # Write log file if configured in environment variable or config file - if config.LOG_LOCATION: - log.setup_file_logger(config.LOG_LOCATION) - - logger.setLevel(logging.__dict__[config.LOG_LEVEL.upper()]) - - # Create a connection to the Tor unix domain control socket or control port - try: - tor_socket = (args.socket or config.TOR_CONTROL_SOCKET) - tor_address = (args.ip or config.TOR_ADDRESS) - tor_port = (args.port or config.TOR_PORT) - try: - controller = Controller.from_socket_file(path=tor_socket) - logger.debug("Successfully connected to the Tor control socket " - "%s.", tor_socket) - except stem.SocketError: - logger.debug("Unable to connect to the Tor control socket %s.", - tor_socket) - controller = Controller.from_port(address=tor_address, - port=tor_port) - logger.debug("Successfully connected to the Tor control port.") - except stem.SocketError as exc: - logger.error("Unable to connect to Tor control socket or port: %s", - exc) - sys.exit(1) - - try: - controller.authenticate(password=config.TOR_CONTROL_PASSWORD) - except stem.connection.AuthenticationFailure as exc: - logger.error("Unable to authenticate on the Tor control connection: " - "%s", exc) - sys.exit(1) - else: - logger.debug("Successfully authenticated on the Tor control " - "connection.") - - status_socket = status.StatusSocket(config.STATUS_SOCKET_LOCATION) - eventhandler.SignalHandler(controller, status_socket) - - # Disable no-member due to bug with "Instance of 'Enum' has no * member" - # pylint: disable=no-member - - # Check that the Tor client supports the HSPOST control port command - if not controller.get_version() >= stem.version.Requirement.HSPOST: - logger.error("A Tor version >= %s is required. You may need to " - "compile Tor from source or install a package from " - "the experimental Tor repository.", - stem.version.Requirement.HSPOST) - sys.exit(1) - - # Load the keys and config for each onion service - settings.initialize_services(controller, - config_file_options.get('services')) - - # Finished parsing all the config file. - - handler = eventhandler.EventHandler() - controller.add_event_listener(handler.new_status, - EventType.STATUS_GENERAL) - controller.add_event_listener(handler.new_desc, - EventType.HS_DESC) - controller.add_event_listener(handler.new_desc_content, - EventType.HS_DESC_CONTENT) - - # Schedule descriptor fetch and upload events - scheduler.add_job(config.REFRESH_INTERVAL, fetch_instance_descriptors, - controller) - scheduler.add_job(config.PUBLISH_CHECK_INTERVAL, publish_all_descriptors) - - # Run initial fetch of HS instance descriptors - scheduler.run_all(delay_seconds=config.INITIAL_DELAY) - - # Begin main loop to poll for HS descriptors - scheduler.run_forever() - - return 0 diff --git a/onionbalance/settings.py b/onionbalance/settings.py deleted file mode 100644 index 7a4d37a..0000000 --- a/onionbalance/settings.py +++ /dev/null @@ -1,380 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -Implements the generation and loading of configuration files. -""" -from builtins import input, range -import os -import sys -import errno -import argparse -import getpass -import logging -import pkg_resources - -import yaml -import Crypto.PublicKey - -from onionbalance import config -from onionbalance import util -from onionbalance import log - -import onionbalance.service -import onionbalance.instance - -logger = log.get_logger() - - -def parse_config_file(config_file): - """ - Parse config file containing service information - """ - config_path = os.path.abspath(config_file) - if os.path.exists(config_path): - with open(config_file, 'r') as handle: - config_data = yaml.load(handle.read()) - logger.info("Loaded the config file '%s'.", config_path) - else: - logger.error("The specified config file '%s' does not exist. The " - "onionbalance-config tool can generate the required " - "keys and config files.", config_path) - sys.exit(1) - - # Rewrite relative paths in the config to be relative to the config - # file directory - config_directory = os.path.dirname(config_path) - for service in config_data.get('services'): - if not os.path.isabs(service.get('key')): - service['key'] = os.path.join(config_directory, service['key']) - - return config_data - - -def initialize_services(controller, services_config): - """ - Load keys for services listed in the config - """ - - # Load the keys and config for each onion service - for service in services_config: - try: - service_key = util.key_decrypt_prompt(service.get("key")) - except (IOError, OSError) as e: - if e.errno == errno.ENOENT: - logger.error("Private key file %s could not be found. " - "Relative paths in the config file are loaded " - "relative to the config file directory.", - service.get("key")) - sys.exit(1) - elif e.errno == errno.EACCES: - logger.error("Permission denied to private key %s.", - service.get("key")) - sys.exit(1) - else: - raise - # Key file was read but a valid private key was not found. - if not service_key: - logger.error("Private key %s could not be loaded. It is a not " - "valid 1024 bit PEM encoded RSA private key", - service.get("key")) - sys.exit(1) - else: - # Successfully imported the private key - onion_address = util.calc_onion_address(service_key) - logger.debug("Loaded private key for service %s.onion.", - onion_address) - - # Load all instances for the current onion service - instance_config = service.get("instances", []) - if not instance_config: - logger.error("Could not load any instances for service " - "%s.onion.", onion_address) - sys.exit(1) - else: - instances = [] - for instance in instance_config: - instances.append(onionbalance.instance.Instance( - controller=controller, - onion_address=instance.get("address"), - authentication_cookie=instance.get("auth") - )) - - logger.info("Loaded %d instances for service %s.onion.", - len(instances), onion_address) - - # Store service configuration in config.services global - config.services.append(onionbalance.service.Service( - controller=controller, - service_key=service_key, - instances=instances - )) - - # Store a global reference to current controller connection - config.controller = controller - - -def parse_cmd_args(): - """ - Parses and returns command line arguments for config generator - """ - - parser = argparse.ArgumentParser( - description="onionbalance-config generates config files and keys for " - "OnionBalance instances and management servers. Calling without any " - "options will initiate an interactive mode.") - - parser.add_argument("--key", type=str, default=None, - help="RSA private key for the master onion service.") - - parser.add_argument("-p", "--password", type=str, default=None, - help="Optional password which can be used to encrypt" - "the master service private key.") - - parser.add_argument("-n", type=int, default=2, dest="num_instances", - help="Number of instances to generate (default: " - "%(default)s).") - - parser.add_argument("-t", "--tag", type=str, default='srv', - help="Prefix name for the service instances " - "(default: %(default)s).") - - parser.add_argument("--output", type=str, default='config/', - help="Directory to store generate config files. " - "The directory will be created if it does not " - "already exist.") - - parser.add_argument("--no-interactive", action='store_true', - help="Try to run automatically without prompting for" - "user input.") - - parser.add_argument("-v", type=str, default="info", dest='verbosity', - help="Minimum verbosity level for logging. Available " - "in ascending order: debug, info, warning, error, " - "critical). The default is info.") - - parser.add_argument("--service-virtual-port", type=str, - default="80", - help="Onion service port for external client " - "connections (default: %(default)s).") - - # TODO: Add validator to check if the target host:port line makes sense. - parser.add_argument("--service-target", type=str, - default="127.0.0.1:80", - help="Target IP and port where your service is " - "listening (default: %(default)s).") - - # .. todo:: Add option to specify HS host and port for instance torrc - - parser.add_argument('--version', action='version', - version='onionbalance %s' % onionbalance.__version__) - - return parser - - -def generate_config(): - """ - Entry point for interactive config file generation. - """ - - # Parse initial command line options - args = parse_cmd_args().parse_args() - - # Simplify the logging output for the command line tool - logger = log.get_config_generator_logger() - - logger.info("Beginning OnionBalance config generation.") - - # If CLI options have been provided, don't enter interactive mode - # Crude check to see if any options beside --verbosity are set. - verbose = True if '-v' in sys.argv else False - - if ((len(sys.argv) > 1 and not verbose) or len(sys.argv) > 3 or - args.no_interactive): - interactive = False - logger.info("Entering non-interactive mode.") - else: - interactive = True - logger.info("No command line arguments found, entering interactive " - "mode.") - - logger.setLevel(logging.__dict__[args.verbosity.upper()]) - - # Check if output directory exists, if not try create it - output_path = None - if interactive: - output_path = input("Enter path to store generated config " - "[{}]: ".format(os.path.abspath(args.output))) - output_path = output_path or args.output - try: - util.try_make_dir(output_path) - except OSError: - logger.exception("Problem encountered when trying to create the " - "output directory %s.", os.path.abspath(output_path)) - else: - logger.debug("Created the output directory '%s'.", - os.path.abspath(output_path)) - - # The output directory should be empty to avoid having conflict keys - # or config files. - if not util.is_directory_empty(output_path): - logger.error("The specified output directory is not empty. Please " - "delete any files and folders or specify another output " - "directory.") - sys.exit(1) - - # Load master key if specified - key_path = None - if interactive: - # Read key path from user - key_path = input("Enter path to master service private key " - "(Leave empty to generate a key): ") - key_path = args.key or key_path - if key_path: - if not os.path.isfile(key_path): - logger.error("The specified master service private key '%s' " - "could not be found. Please confirm the path and " - "file permissions are correct.", key_path) - sys.exit(1) - else: - # Try load the specified private key file - master_key = util.key_decrypt_prompt(key_path) - if not master_key: - logger.error("The specified master private key %s could not " - "be loaded.", os.path.abspath(master_key)) - sys.exit(1) - else: - master_onion_address = util.calc_onion_address(master_key) - logger.info("Successfully loaded a master key for service " - "%s.onion.", master_onion_address) - - else: - # No key specified, begin generating a new one. - master_key = Crypto.PublicKey.RSA.generate(1024) - master_onion_address = util.calc_onion_address(master_key) - logger.debug("Created a new master key for service %s.onion.", - master_onion_address) - - # Finished loading/generating master key, now try generate keys for - # each service instance - num_instances = None - if interactive: - num_instances = input("Number of instance services to create " - "[{}]: ".format(args.num_instances)) - # Cast to int if a number was specified - try: - num_instances = int(num_instances) - except ValueError: - num_instances = None - num_instances = num_instances or args.num_instances - logger.debug("Creating %d service instances.", num_instances) - - tag = None - if interactive: - tag = input("Provide a tag name to group these instances " - "[{}]: ".format(args.tag)) - tag = tag or args.tag - - # Create HiddenServicePort line for instance torrc file - service_virtual_port = None - if interactive: - service_virtual_port = input("Specify the service virtual port (for " - "client connections) [{}]: ".format( - args.service_virtual_port)) - service_virtual_port = service_virtual_port or args.service_virtual_port - - service_target = None - if interactive: - # In interactive mode, change default target to match the specified - # virtual port - default_service_target = u'127.0.0.1:{}'.format(service_virtual_port) - service_target = input("Specify the service target IP and port (where " - "your service is listening) [{}]: ".format( - default_service_target)) - service_target = service_target or default_service_target - service_target = service_target or args.service_target - torrc_port_line = u'HiddenServicePort {} {}'.format(service_virtual_port, - service_target) - - instances = [] - for i in range(0, num_instances): - instance_key = Crypto.PublicKey.RSA.generate(1024) - instance_address = util.calc_onion_address(instance_key) - logger.debug("Created a key for instance %s.onion.", - instance_address) - instances.append((instance_address, instance_key)) - - # Write master service key to directory - master_passphrase = None - if interactive: - master_passphrase = getpass.getpass( - "Provide an optional password to encrypt the master private " - "key (Not encrypted if no password is specified): ") - master_passphrase = master_passphrase or args.password - - # Finished reading input, starting to write config files. - master_dir = os.path.join(output_path, 'master') - util.try_make_dir(master_dir) - master_key_file = os.path.join(master_dir, - '{}.key'.format(master_onion_address)) - with open(master_key_file, "wb") as key_file: - os.chmod(master_key_file, 384) # chmod 0600 in decimal - key_file.write(master_key.exportKey(passphrase=master_passphrase)) - logger.debug("Successfully wrote master key to file %s.", - os.path.abspath(master_key_file)) - - # Create YAML OnionBalance settings file for these instances - service_data = {'key': '{}.key'.format(master_onion_address)} - service_data['instances'] = [{'address': address, - 'name': '{}{}'.format(tag, i+1)} for - i, (address, _) in enumerate(instances)] - settings_data = {'services': [service_data]} - config_yaml = yaml.safe_dump(settings_data, default_flow_style=False) - - config_file_path = os.path.join(master_dir, 'config.yaml') - with open(config_file_path, "w") as config_file: - config_file.write(u"# OnionBalance Config File\n") - config_file.write(config_yaml) - logger.info("Wrote master service config file '%s'.", - os.path.abspath(config_file_path)) - - # Write master service torrc - master_torrc_path = os.path.join(master_dir, 'torrc-server') - master_torrc_template = pkg_resources.resource_string(__name__, - 'data/torrc-server') - with open(master_torrc_path, "w") as master_torrc_file: - master_torrc_file.write(master_torrc_template.decode('utf-8')) - - # Try generate config files for each service instance - for i, (instance_address, instance_key) in enumerate(instances): - # Create a numbered directory for instance - instance_dir = os.path.join(output_path, '{}{}'.format(tag, i+1)) - instance_key_dir = os.path.join(instance_dir, instance_address) - util.try_make_dir(instance_key_dir) - os.chmod(instance_key_dir, 1472) # chmod 2700 in decimal - - instance_key_file = os.path.join(instance_key_dir, 'private_key') - with open(instance_key_file, "wb") as key_file: - os.chmod(instance_key_file, 384) # chmod 0600 in decimal - key_file.write(instance_key.exportKey()) - logger.debug("Successfully wrote key for instance %s.onion to " - "file.", instance_address) - - # Write torrc file for each instance - instance_torrc = os.path.join(instance_dir, 'instance_torrc') - instance_torrc_template = pkg_resources.resource_string( - __name__, 'data/torrc-instance') - with open(instance_torrc, "w") as torrc_file: - torrc_file.write(instance_torrc_template.decode('utf-8')) - # The ./ relative path prevents Tor from raising relative - # path warnings. The relative path may need to be edited manual - # to work on Windows systems. - torrc_file.write(u"HiddenServiceDir {}\n".format( - instance_address)) - torrc_file.write(u"{}\n".format(torrc_port_line)) - - # Output final status message - logger.info("Done! Successfully generated an OnionBalance config and %d " - "instance keys for service %s.onion.", - num_instances, master_onion_address) - - sys.exit(0) diff --git a/setup.cfg b/setup.cfg index 4562916..58f38ef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,13 @@ -[pytest] +[tool:pytest] norecursedirs = .tox _build tor chutney [bdist_wheel] universal=1 + +[versioneer] +VCS = git +style = pep440 +versionfile_source = onionbalance/_version.py +versionfile_build = None +tag_prefix ='' +parentdir_prefix = onionbalance- \ No newline at end of file diff --git a/setup.py b/setup.py index 11419ae..32dd558 100644 --- a/setup.py +++ b/setup.py @@ -3,6 +3,7 @@ """setup.py: setuptools control.""" import io +import versioneer import os from setuptools import setup @@ -23,17 +24,18 @@ def read(*names, **kwargs): setup( name="OnionBalance", - packages=["onionbalance"], + packages=["onionbalance", + "onionbalance.hs_v2", "onionbalance.hs_v3", "onionbalance.common", + "onionbalance.config_generator"], entry_points={ "console_scripts": [ - 'onionbalance = onionbalance.manager:main', - 'onionbalance-config = onionbalance.settings:generate_config', + 'onionbalance = onionbalance.hs_v2.manager:main', # XXX + 'onionbalance-config = onionbalance.config_generator.config_generator:main', ]}, description="OnionBalance provides load-balancing and redundancy for Tor " "hidden services by distributing requests to multiple backend " "Tor instances.", long_description=read('README.rst'), - version=module_info.get('__version__'), author=module_info.get('__author__'), author_email=module_info.get('__contact__'), url=module_info.get('__url__'), @@ -48,7 +50,7 @@ def read(*names, **kwargs): 'setproctitle', ], tests_require=['tox', 'pytest-mock', 'pytest', 'mock', 'pexpect'], - package_data={'onionbalance': ['data/*']}, + package_data={'onionbalance.config_generator': ['data/*']}, include_package_data=True, classifiers=[ 'Development Status :: 3 - Alpha', @@ -57,5 +59,7 @@ def read(*names, **kwargs): 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', - ] + ], + version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), ) diff --git a/test/functional/test_onionbalance_config.py b/test/functional/test_onionbalance_config.py index 07ed673..8906002 100644 --- a/test/functional/test_onionbalance_config.py +++ b/test/functional/test_onionbalance_config.py @@ -8,13 +8,16 @@ import pexpect import Crypto.PublicKey.RSA -import onionbalance.util +import onionbalance.hs_v2.util def onionbalance_config_interact(cli, cli_input): """ Send each input line to the onionbalance-config CLI interface """ + cli.expect(u"Enter HS version") + cli.send("v2\n") + cli.expect(u"store generated config") cli.send("{}\n".format(cli_input.get('config_dir', u''))) @@ -93,6 +96,7 @@ def test_onionbalance_config_automatic(tmpdir): # Start onionbalance-config in automatic mode cli = pexpect.spawnu("onionbalance-config", logfile=sys.stdout, args=[ + '--hs-version', 'v2', '--output', str(tmpdir.join(u"configdir")), ]) cli.expect(u"Done! Successfully generated") @@ -106,6 +110,7 @@ def test_onionbalance_config_automatic_custom_ports(tmpdir): """ cli = pexpect.spawnu("onionbalance-config", logfile=sys.stdout, args=[ + '--hs-version', 'v2', '--output', str(tmpdir.join(u"configdir")), '--service-virtual-port', u'443', '--service-target', u'127.0.0.1:8443', @@ -139,6 +144,7 @@ def test_onionbalance_config_automatic_key_with_password(tmpdir, mocker): # Start onionbalance-config in automatic mode cli = pexpect.spawnu("onionbalance-config", logfile=sys.stdout, args=[ + '--hs-version', 'v2', '--output', str(tmpdir.join(u"configdir")), '--key', str(key_path), '--password', 'testpassword', @@ -153,5 +159,5 @@ def test_onionbalance_config_automatic_key_with_password(tmpdir, mocker): # Check key decrypts and is valid mocker.patch('getpass.getpass', lambda *_: 'testpassword') - output_key = onionbalance.util.key_decrypt_prompt(str(output_key_path)) + output_key = onionbalance.hs_v2.util.key_decrypt_prompt(str(output_key_path)) assert isinstance(output_key, Crypto.PublicKey.RSA._RSAobj) diff --git a/test/functional/test_publish_master_descriptor.py b/test/functional/test_publish_master_descriptor.py index 87e444e..09a44da 100644 --- a/test/functional/test_publish_master_descriptor.py +++ b/test/functional/test_publish_master_descriptor.py @@ -10,7 +10,7 @@ import pexpect import stem.control -import onionbalance.util +import onionbalance.hs_v2.util # Skip functional tests if Chutney environment is not running. pytestmark = pytest.mark.skipif( @@ -91,7 +91,7 @@ def test_master_descriptor_publication(tmpdir): chutney_config = parse_chutney_enviroment() private_key = Crypto.PublicKey.RSA.generate(1024) - master_onion_address = onionbalance.util.calc_onion_address(private_key) + master_onion_address = onionbalance.hs_v2.util.calc_onion_address(private_key) config_file_path = create_test_config_file( tmppath=tmpdir, diff --git a/test/scripts/install-chutney.sh b/test/scripts/install-chutney-v2.sh similarity index 100% rename from test/scripts/install-chutney.sh rename to test/scripts/install-chutney-v2.sh diff --git a/test/v2/__init__.py b/test/v2/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_consensus.py b/test/v2/test_consensus.py similarity index 96% rename from test/test_consensus.py rename to test/v2/test_consensus.py index 4ba41cf..dbe9d25 100644 --- a/test/test_consensus.py +++ b/test/v2/test_consensus.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- import pytest -from onionbalance import consensus -from onionbalance import config +from onionbalance.hs_v2 import consensus +from onionbalance.hs_v2 import config # Mock hex-encoded HSDir fingerprint list MOCK_HSDIR_LIST = [ diff --git a/test/test_descriptor.py b/test/v2/test_descriptor.py similarity index 97% rename from test/test_descriptor.py rename to test/v2/test_descriptor.py index eec868e..1e6cd59 100644 --- a/test/test_descriptor.py +++ b/test/v2/test_descriptor.py @@ -5,11 +5,13 @@ import pytest import Crypto.PublicKey.RSA import stem.descriptor +import stem.descriptor.hidden_service_descriptor import hashlib from binascii import unhexlify -from onionbalance import descriptor +from onionbalance.common import intro_point_set +from onionbalance.hs_v2 import descriptor PEM_PRIVATE_KEY = u'\n'.join([ '-----BEGIN RSA PRIVATE KEY-----', @@ -201,6 +203,8 @@ UNIX_TIMESTAMP = 1435233021 +""" +TODO: Reenable test that fails with Pytest3 @pytest.mark.parametrize('intro_point_distribution, selected_ip_count', [ ([3], 3), ([3, 3], 6), @@ -213,23 +217,21 @@ ]) def test_introduction_point_selection(intro_point_distribution, selected_ip_count): - """ - Basic test case to check that the correct number of IPs are selected. - """ + # Basic test case to check that the correct number of IPs are selected. # Create Mock list of instances (index by letter) and their respective # introduction points. available_intro_points = [[index] * count for index, count in zip(string.ascii_lowercase, intro_point_distribution)] - intro_set = descriptor.IntroductionPointSet(available_intro_points) + intro_set = intro_point_set.IntroductionPointSet(available_intro_points) # Check that we can fetch the same number for each descriptor for i in range(0, 2): # Max 10 introduction points per descriptor choosen_intro_points = intro_set.choose(10) assert len(choosen_intro_points) == selected_ip_count - +""" def test_generate_service_descriptor(monkeypatch, mocker): """ @@ -244,7 +246,7 @@ def utcnow(cls): # Patch make_introduction_points_part to return the test introduction # point section - mocker.patch('onionbalance.descriptor.make_introduction_points_part', + mocker.patch('onionbalance.hs_v2.descriptor.make_introduction_points_part', lambda *_: INTRODUCTION_POINT_PART) # Test basic descriptor generation. @@ -334,7 +336,7 @@ def test_descriptor_received_invalid_descriptor(mocker): """ Test invalid descriptor content received from the HSDir """ - mocker.patch("onionbalance.descriptor.logger.exception", + mocker.patch("onionbalance.hs_v2.descriptor.logger.exception", side_effect=ValueError('InvalidDescriptorException')) # Check that the invalid descriptor error is logged. diff --git a/test/test_settings.py b/test/v2/test_settings.py similarity index 97% rename from test/test_settings.py rename to test/v2/test_settings.py index f987803..42fd271 100644 --- a/test/test_settings.py +++ b/test/v2/test_settings.py @@ -4,7 +4,7 @@ import pytest -from onionbalance import settings +from onionbalance.hs_v2 import settings from .util import builtin CONFIG_FILE_VALID = u'\n'.join([ diff --git a/test/test_util.py b/test/v2/test_util.py similarity index 99% rename from test/test_util.py rename to test/v2/test_util.py index 39f9ec3..1ecffae 100644 --- a/test/test_util.py +++ b/test/v2/test_util.py @@ -10,7 +10,7 @@ import pytest from .util import builtin -from onionbalance.util import * +from onionbalance.hs_v2.util import * PEM_PRIVATE_KEY = u'\n'.join([ diff --git a/test/util.py b/test/v2/util.py similarity index 100% rename from test/util.py rename to test/v2/util.py diff --git a/test/v3/test_v3_descriptor.py b/test/v3/test_v3_descriptor.py new file mode 100644 index 0000000..bb26273 --- /dev/null +++ b/test/v3/test_v3_descriptor.py @@ -0,0 +1,22 @@ +from onionbalance.hs_v3 import consensus + +def test_disaster_srv(): + """ + Test that disaster SRV creation is correct based on little-t-tor's + unittests as test vectors (in particular see test_disaster_srv()) + """ + my_consensus = consensus.Consensus(do_refresh_consensus=False) + + # Correct disaster SRV for TPs: 1, 2, 3, 4, 5 + correct_srvs = [ "F8A4948707653837FA44ABB5BBC75A12F6F101E7F8FAF699B9715F4965D3507D", + "C17966DF8B4834638E1B7BF38944C4995B92E89749D1623E417A44938D08FD67", + "A3BAB4327F9C2F8B30A126DAD3ABCCE12DD813169A5D924244B57987AEE413C2", + "C79ABE7116FCF7810F7A5C76198B97339990C738B456EFDBC1399189927BADEC", + "941D5FE4289FBF2F853766EAC4B948B51C81A4137A44516342ABC80518E0183D"] + + for i in range(1,6): + disaster_srv = my_consensus._get_disaster_srv(i) + assert(disaster_srv.hex().upper() == correct_srvs[i-1]) + +if __name__ == '__main__': + unittest.main() diff --git a/test/v3/test_v3_hashring.py b/test/v3/test_v3_hashring.py new file mode 100644 index 0000000..67c8118 --- /dev/null +++ b/test/v3/test_v3_hashring.py @@ -0,0 +1,100 @@ +import unittest +import mock +import datetime +import base64 + +from cryptography.hazmat.primitives.asymmetric import ed25519 + +from onionbalance.hs_v3 import tor_node +from onionbalance.hs_v3 import hashring +from onionbalance.hs_v3 import consensus + +CORRECT_HSDIR_FPRS_FIRST_DESCRIPTOR = [ + "D1D1D1D1D1D1D1D1D1D1D1D1D1D1D1D1D1D1D1D1", + "2F2F2F2F2F2F2F2F2F2F2F2F2F2F2F2F2F2F2F2F", + "B0B0B0B0B0B0B0B0B0B0B0B0B0B0B0B0B0B0B0B0", + "3A3A3A3A3A3A3A3A3A3A3A3A3A3A3A3A3A3A3A3A", + "5A5A5A5A5A5A5A5A5A5A5A5A5A5A5A5A5A5A5A5A", + "DFDFDFDFDFDFDFDFDFDFDFDFDFDFDFDFDFDFDFDF", + "F7F7F7F7F7F7F7F7F7F7F7F7F7F7F7F7F7F7F7F7", + "3434343434343434343434343434343434343434" ] + +CORRECT_HSDIR_FPRS_SECOND_DESCRIPTOR = [ + "5D5D5D5D5D5D5D5D5D5D5D5D5D5D5D5D5D5D5D5D", + "9A9A9A9A9A9A9A9A9A9A9A9A9A9A9A9A9A9A9A9A", + "D1D1D1D1D1D1D1D1D1D1D1D1D1D1D1D1D1D1D1D1", + "7A7A7A7A7A7A7A7A7A7A7A7A7A7A7A7A7A7A7A7A", + "C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3C3", + "C6C6C6C6C6C6C6C6C6C6C6C6C6C6C6C6C6C6C6C6", + "E9E9E9E9E9E9E9E9E9E9E9E9E9E9E9E9E9E9E9E9", + "8686868686868686868686868686868686868686" +] + +class DummyConsensus(consensus.Consensus): + def __init__(self): + self.consensus = None + + +class TestHashRing(unittest.TestCase): + def test_hashring(self): + current_time = datetime.datetime.fromtimestamp(10101010101) + current_srv = bytes([41])*32 + previous_srv = bytes([42])*32 + + # Create 255 fake Tor nodes that will be used as part of the unittest + network_nodes = [] + for i in range(1,256): + microdescriptor = mock.Mock() + routerstatus = mock.Mock() + + routerstatus.fingerprint = (bytes([i])*20).hex() + routerstatus.protocols = {'HSDir' : [2]} + routerstatus.flags = ['HSDir'] + node_ed25519_id_b64 = base64.b64encode(bytes([i])*32).decode('utf-8') + microdescriptor.identifiers = {'ed25519' : node_ed25519_id_b64} + node = tor_node.Node(microdescriptor, routerstatus) + network_nodes.append(node) + + # Mock a fake consensus + consensus = DummyConsensus() + consensus.consensus = mock.Mock() + consensus.consensus.valid_after = current_time + consensus.get_current_srv = mock.Mock() + consensus.get_current_srv.return_value = current_srv + consensus.get_previous_srv = mock.Mock() + consensus.get_previous_srv.return_value = previous_srv + consensus.is_live = mock.Mock() + consensus.is_live.return_value = True + consensus.nodes = network_nodes + + # Mock a fake Tor network + from onionbalance.hs_v3.onionbalance import my_onionbalance + my_onionbalance.consensus = consensus + + hash_ring = hashring.HSV3HashRing() + + previous_blinded_pubkey_hex = "063AEC5E1FD3025098F2DF71EF570B28D94B463FFCCB5EC6A9C061E38F551C6A" + previous_blinded_pubkey_bytes = base64.b16decode(previous_blinded_pubkey_hex) + + responsible_hsdirs = hash_ring.get_responsible_hsdirs(previous_blinded_pubkey_bytes, True) + + i = 0 + for responsible_hsdir in responsible_hsdirs: + self.assertEqual(responsible_hsdir.upper(), CORRECT_HSDIR_FPRS_FIRST_DESCRIPTOR[i]) + i+=1 + + print("===") + + # we need to use the new blinded key since this uses a new time period......... + current_blinded_pubkey_hex = "5DB624F2D74F103E6E8C6FBCCD074586EF5A5572F90673C00B77DEF94EC11499" + current_blinded_pubkey_bytes = base64.b16decode(current_blinded_pubkey_hex) + + responsible_hsdirs = hash_ring.get_responsible_hsdirs(current_blinded_pubkey_bytes, False) + + i = 0 + for responsible_hsdir in responsible_hsdirs: + self.assertEqual(responsible_hsdir.upper(), CORRECT_HSDIR_FPRS_SECOND_DESCRIPTOR[i]) + i+=1 + +if __name__ == '__main__': + unittest.main() diff --git a/versioneer.py b/versioneer.py new file mode 100644 index 0000000..64fea1c --- /dev/null +++ b/versioneer.py @@ -0,0 +1,1822 @@ + +# Version: 0.18 + +"""The Versioneer - like a rocketeer, but for versions. + +The Versioneer +============== + +* like a rocketeer, but for versions! +* https://github.com/warner/python-versioneer +* Brian Warner +* License: Public Domain +* Compatible With: python2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, and pypy +* [![Latest Version] +(https://pypip.in/version/versioneer/badge.svg?style=flat) +](https://pypi.python.org/pypi/versioneer/) +* [![Build Status] +(https://travis-ci.org/warner/python-versioneer.png?branch=master) +](https://travis-ci.org/warner/python-versioneer) + +This is a tool for managing a recorded version number in distutils-based +python projects. The goal is to remove the tedious and error-prone "update +the embedded version string" step from your release process. Making a new +release should be as easy as recording a new tag in your version-control +system, and maybe making new tarballs. + + +## Quick Install + +* `pip install versioneer` to somewhere to your $PATH +* add a `[versioneer]` section to your setup.cfg (see below) +* run `versioneer install` in your source tree, commit the results + +## Version Identifiers + +Source trees come from a variety of places: + +* a version-control system checkout (mostly used by developers) +* a nightly tarball, produced by build automation +* a snapshot tarball, produced by a web-based VCS browser, like github's + "tarball from tag" feature +* a release tarball, produced by "setup.py sdist", distributed through PyPI + +Within each source tree, the version identifier (either a string or a number, +this tool is format-agnostic) can come from a variety of places: + +* ask the VCS tool itself, e.g. "git describe" (for checkouts), which knows + about recent "tags" and an absolute revision-id +* the name of the directory into which the tarball was unpacked +* an expanded VCS keyword ($Id$, etc) +* a `_version.py` created by some earlier build step + +For released software, the version identifier is closely related to a VCS +tag. Some projects use tag names that include more than just the version +string (e.g. "myproject-1.2" instead of just "1.2"), in which case the tool +needs to strip the tag prefix to extract the version identifier. For +unreleased software (between tags), the version identifier should provide +enough information to help developers recreate the same tree, while also +giving them an idea of roughly how old the tree is (after version 1.2, before +version 1.3). Many VCS systems can report a description that captures this, +for example `git describe --tags --dirty --always` reports things like +"0.7-1-g574ab98-dirty" to indicate that the checkout is one revision past the +0.7 tag, has a unique revision id of "574ab98", and is "dirty" (it has +uncommitted changes. + +The version identifier is used for multiple purposes: + +* to allow the module to self-identify its version: `myproject.__version__` +* to choose a name and prefix for a 'setup.py sdist' tarball + +## Theory of Operation + +Versioneer works by adding a special `_version.py` file into your source +tree, where your `__init__.py` can import it. This `_version.py` knows how to +dynamically ask the VCS tool for version information at import time. + +`_version.py` also contains `$Revision$` markers, and the installation +process marks `_version.py` to have this marker rewritten with a tag name +during the `git archive` command. As a result, generated tarballs will +contain enough information to get the proper version. + +To allow `setup.py` to compute a version too, a `versioneer.py` is added to +the top level of your source tree, next to `setup.py` and the `setup.cfg` +that configures it. This overrides several distutils/setuptools commands to +compute the version when invoked, and changes `setup.py build` and `setup.py +sdist` to replace `_version.py` with a small static file that contains just +the generated version data. + +## Installation + +See [INSTALL.md](./INSTALL.md) for detailed installation instructions. + +## Version-String Flavors + +Code which uses Versioneer can learn about its version string at runtime by +importing `_version` from your main `__init__.py` file and running the +`get_versions()` function. From the "outside" (e.g. in `setup.py`), you can +import the top-level `versioneer.py` and run `get_versions()`. + +Both functions return a dictionary with different flavors of version +information: + +* `['version']`: A condensed version string, rendered using the selected + style. This is the most commonly used value for the project's version + string. The default "pep440" style yields strings like `0.11`, + `0.11+2.g1076c97`, or `0.11+2.g1076c97.dirty`. See the "Styles" section + below for alternative styles. + +* `['full-revisionid']`: detailed revision identifier. For Git, this is the + full SHA1 commit id, e.g. "1076c978a8d3cfc70f408fe5974aa6c092c949ac". + +* `['date']`: Date and time of the latest `HEAD` commit. For Git, it is the + commit date in ISO 8601 format. This will be None if the date is not + available. + +* `['dirty']`: a boolean, True if the tree has uncommitted changes. Note that + this is only accurate if run in a VCS checkout, otherwise it is likely to + be False or None + +* `['error']`: if the version string could not be computed, this will be set + to a string describing the problem, otherwise it will be None. It may be + useful to throw an exception in setup.py if this is set, to avoid e.g. + creating tarballs with a version string of "unknown". + +Some variants are more useful than others. Including `full-revisionid` in a +bug report should allow developers to reconstruct the exact code being tested +(or indicate the presence of local changes that should be shared with the +developers). `version` is suitable for display in an "about" box or a CLI +`--version` output: it can be easily compared against release notes and lists +of bugs fixed in various releases. + +The installer adds the following text to your `__init__.py` to place a basic +version in `YOURPROJECT.__version__`: + + from ._version import get_versions + __version__ = get_versions()['version'] + del get_versions + +## Styles + +The setup.cfg `style=` configuration controls how the VCS information is +rendered into a version string. + +The default style, "pep440", produces a PEP440-compliant string, equal to the +un-prefixed tag name for actual releases, and containing an additional "local +version" section with more detail for in-between builds. For Git, this is +TAG[+DISTANCE.gHEX[.dirty]] , using information from `git describe --tags +--dirty --always`. For example "0.11+2.g1076c97.dirty" indicates that the +tree is like the "1076c97" commit but has uncommitted changes (".dirty"), and +that this commit is two revisions ("+2") beyond the "0.11" tag. For released +software (exactly equal to a known tag), the identifier will only contain the +stripped tag, e.g. "0.11". + +Other styles are available. See [details.md](details.md) in the Versioneer +source tree for descriptions. + +## Debugging + +Versioneer tries to avoid fatal errors: if something goes wrong, it will tend +to return a version of "0+unknown". To investigate the problem, run `setup.py +version`, which will run the version-lookup code in a verbose mode, and will +display the full contents of `get_versions()` (including the `error` string, +which may help identify what went wrong). + +## Known Limitations + +Some situations are known to cause problems for Versioneer. This details the +most significant ones. More can be found on Github +[issues page](https://github.com/warner/python-versioneer/issues). + +### Subprojects + +Versioneer has limited support for source trees in which `setup.py` is not in +the root directory (e.g. `setup.py` and `.git/` are *not* siblings). The are +two common reasons why `setup.py` might not be in the root: + +* Source trees which contain multiple subprojects, such as + [Buildbot](https://github.com/buildbot/buildbot), which contains both + "master" and "slave" subprojects, each with their own `setup.py`, + `setup.cfg`, and `tox.ini`. Projects like these produce multiple PyPI + distributions (and upload multiple independently-installable tarballs). +* Source trees whose main purpose is to contain a C library, but which also + provide bindings to Python (and perhaps other langauges) in subdirectories. + +Versioneer will look for `.git` in parent directories, and most operations +should get the right version string. However `pip` and `setuptools` have bugs +and implementation details which frequently cause `pip install .` from a +subproject directory to fail to find a correct version string (so it usually +defaults to `0+unknown`). + +`pip install --editable .` should work correctly. `setup.py install` might +work too. + +Pip-8.1.1 is known to have this problem, but hopefully it will get fixed in +some later version. + +[Bug #38](https://github.com/warner/python-versioneer/issues/38) is tracking +this issue. The discussion in +[PR #61](https://github.com/warner/python-versioneer/pull/61) describes the +issue from the Versioneer side in more detail. +[pip PR#3176](https://github.com/pypa/pip/pull/3176) and +[pip PR#3615](https://github.com/pypa/pip/pull/3615) contain work to improve +pip to let Versioneer work correctly. + +Versioneer-0.16 and earlier only looked for a `.git` directory next to the +`setup.cfg`, so subprojects were completely unsupported with those releases. + +### Editable installs with setuptools <= 18.5 + +`setup.py develop` and `pip install --editable .` allow you to install a +project into a virtualenv once, then continue editing the source code (and +test) without re-installing after every change. + +"Entry-point scripts" (`setup(entry_points={"console_scripts": ..})`) are a +convenient way to specify executable scripts that should be installed along +with the python package. + +These both work as expected when using modern setuptools. When using +setuptools-18.5 or earlier, however, certain operations will cause +`pkg_resources.DistributionNotFound` errors when running the entrypoint +script, which must be resolved by re-installing the package. This happens +when the install happens with one version, then the egg_info data is +regenerated while a different version is checked out. Many setup.py commands +cause egg_info to be rebuilt (including `sdist`, `wheel`, and installing into +a different virtualenv), so this can be surprising. + +[Bug #83](https://github.com/warner/python-versioneer/issues/83) describes +this one, but upgrading to a newer version of setuptools should probably +resolve it. + +### Unicode version strings + +While Versioneer works (and is continually tested) with both Python 2 and +Python 3, it is not entirely consistent with bytes-vs-unicode distinctions. +Newer releases probably generate unicode version strings on py2. It's not +clear that this is wrong, but it may be surprising for applications when then +write these strings to a network connection or include them in bytes-oriented +APIs like cryptographic checksums. + +[Bug #71](https://github.com/warner/python-versioneer/issues/71) investigates +this question. + + +## Updating Versioneer + +To upgrade your project to a new release of Versioneer, do the following: + +* install the new Versioneer (`pip install -U versioneer` or equivalent) +* edit `setup.cfg`, if necessary, to include any new configuration settings + indicated by the release notes. See [UPGRADING](./UPGRADING.md) for details. +* re-run `versioneer install` in your source tree, to replace + `SRC/_version.py` +* commit any changed files + +## Future Directions + +This tool is designed to make it easily extended to other version-control +systems: all VCS-specific components are in separate directories like +src/git/ . The top-level `versioneer.py` script is assembled from these +components by running make-versioneer.py . In the future, make-versioneer.py +will take a VCS name as an argument, and will construct a version of +`versioneer.py` that is specific to the given VCS. It might also take the +configuration arguments that are currently provided manually during +installation by editing setup.py . Alternatively, it might go the other +direction and include code from all supported VCS systems, reducing the +number of intermediate scripts. + + +## License + +To make Versioneer easier to embed, all its code is dedicated to the public +domain. The `_version.py` that it creates is also in the public domain. +Specifically, both are released under the Creative Commons "Public Domain +Dedication" license (CC0-1.0), as described in +https://creativecommons.org/publicdomain/zero/1.0/ . + +""" + +from __future__ import print_function +try: + import configparser +except ImportError: + import ConfigParser as configparser +import errno +import json +import os +import re +import subprocess +import sys + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_root(): + """Get the project root directory. + + We require that all commands are run from the project root, i.e. the + directory that contains setup.py, setup.cfg, and versioneer.py . + """ + root = os.path.realpath(os.path.abspath(os.getcwd())) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + # allow 'python path/to/setup.py COMMAND' + root = os.path.dirname(os.path.realpath(os.path.abspath(sys.argv[0]))) + setup_py = os.path.join(root, "setup.py") + versioneer_py = os.path.join(root, "versioneer.py") + if not (os.path.exists(setup_py) or os.path.exists(versioneer_py)): + err = ("Versioneer was unable to run the project root directory. " + "Versioneer requires setup.py to be executed from " + "its immediate directory (like 'python setup.py COMMAND'), " + "or in a way that lets it use sys.argv[0] to find the root " + "(like 'python path/to/setup.py COMMAND').") + raise VersioneerBadRootError(err) + try: + # Certain runtime workflows (setup.py install/develop in a setuptools + # tree) execute all dependencies in a single python process, so + # "versioneer" may be imported multiple times, and python's shared + # module-import table will cache the first one. So we can't use + # os.path.dirname(__file__), as that will find whichever + # versioneer.py was first imported, even in later projects. + me = os.path.realpath(os.path.abspath(__file__)) + me_dir = os.path.normcase(os.path.splitext(me)[0]) + vsr_dir = os.path.normcase(os.path.splitext(versioneer_py)[0]) + if me_dir != vsr_dir: + print("Warning: build in %s is using versioneer.py from %s" + % (os.path.dirname(me), versioneer_py)) + except NameError: + pass + return root + + +def get_config_from_root(root): + """Read the project setup.cfg file to determine Versioneer config.""" + # This might raise EnvironmentError (if setup.cfg is missing), or + # configparser.NoSectionError (if it lacks a [versioneer] section), or + # configparser.NoOptionError (if it lacks "VCS="). See the docstring at + # the top of versioneer.py for instructions on writing your setup.cfg . + setup_cfg = os.path.join(root, "setup.cfg") + parser = configparser.SafeConfigParser() + with open(setup_cfg, "r") as f: + parser.readfp(f) + VCS = parser.get("versioneer", "VCS") # mandatory + + def get(parser, name): + if parser.has_option("versioneer", name): + return parser.get("versioneer", name) + return None + cfg = VersioneerConfig() + cfg.VCS = VCS + cfg.style = get(parser, "style") or "" + cfg.versionfile_source = get(parser, "versionfile_source") + cfg.versionfile_build = get(parser, "versionfile_build") + cfg.tag_prefix = get(parser, "tag_prefix") + if cfg.tag_prefix in ("''", '""'): + cfg.tag_prefix = "" + cfg.parentdir_prefix = get(parser, "parentdir_prefix") + cfg.verbose = get(parser, "verbose") + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +# these dictionaries contain VCS-specific tools +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %s" % dispcmd) + print(e) + return None, None + else: + if verbose: + print("unable to find command, tried %s" % (commands,)) + return None, None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %s (error)" % dispcmd) + print("stdout was %s" % stdout) + return None, p.returncode + return stdout, p.returncode + + +LONG_VERSION_PY['git'] = ''' +# This file helps to compute a version number in source trees obtained from +# git-archive tarball (such as those provided by githubs download-from-tag +# feature). Distribution tarballs (built by setup.py sdist) and build +# directories (produced by setup.py build) will contain a much shorter file +# that just contains the computed version number. + +# This file is released into the public domain. Generated by +# versioneer-0.18 (https://github.com/warner/python-versioneer) + +"""Git implementation of _version.py.""" + +import errno +import os +import re +import subprocess +import sys + + +def get_keywords(): + """Get the keywords needed to look up the version information.""" + # these strings will be replaced by git during git-archive. + # setup.py/versioneer.py will grep for the variable names, so they must + # each be defined on a line of their own. _version.py will just call + # get_keywords(). + git_refnames = "%(DOLLAR)sFormat:%%d%(DOLLAR)s" + git_full = "%(DOLLAR)sFormat:%%H%(DOLLAR)s" + git_date = "%(DOLLAR)sFormat:%%ci%(DOLLAR)s" + keywords = {"refnames": git_refnames, "full": git_full, "date": git_date} + return keywords + + +class VersioneerConfig: + """Container for Versioneer configuration parameters.""" + + +def get_config(): + """Create, populate and return the VersioneerConfig() object.""" + # these strings are filled in when 'setup.py versioneer' creates + # _version.py + cfg = VersioneerConfig() + cfg.VCS = "git" + cfg.style = "%(STYLE)s" + cfg.tag_prefix = "%(TAG_PREFIX)s" + cfg.parentdir_prefix = "%(PARENTDIR_PREFIX)s" + cfg.versionfile_source = "%(VERSIONFILE_SOURCE)s" + cfg.verbose = False + return cfg + + +class NotThisMethod(Exception): + """Exception raised if a method is not valid for the current scenario.""" + + +LONG_VERSION_PY = {} +HANDLERS = {} + + +def register_vcs_handler(vcs, method): # decorator + """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): + """Store f in HANDLERS[vcs][method].""" + if vcs not in HANDLERS: + HANDLERS[vcs] = {} + HANDLERS[vcs][method] = f + return f + return decorate + + +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, + env=None): + """Call the given command(s).""" + assert isinstance(commands, list) + p = None + for c in commands: + try: + dispcmd = str([c] + args) + # remember shell=False, so use git.cmd on windows, not just git + p = subprocess.Popen([c] + args, cwd=cwd, env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr + else None)) + break + except EnvironmentError: + e = sys.exc_info()[1] + if e.errno == errno.ENOENT: + continue + if verbose: + print("unable to run %%s" %% dispcmd) + print(e) + return None, None + else: + if verbose: + print("unable to find command, tried %%s" %% (commands,)) + return None, None + stdout = p.communicate()[0].strip() + if sys.version_info[0] >= 3: + stdout = stdout.decode() + if p.returncode != 0: + if verbose: + print("unable to run %%s (error)" %% dispcmd) + print("stdout was %%s" %% stdout) + return None, p.returncode + return stdout, p.returncode + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %%s but none started with prefix %%s" %% + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # git-2.2.0 added "%%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %%d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%%s', no digits" %% ",".join(refs - tags)) + if verbose: + print("likely tags: %%s" %% ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %%s" %% r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %%s not under git control" %% root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%%s*" %% tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%%s'" + %% describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%%s' doesn't start with prefix '%%s'" + print(fmt %% (full_tag, tag_prefix)) + pieces["error"] = ("tag '%%s' doesn't start with prefix '%%s'" + %% (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%%ci", "HEAD"], + cwd=root)[0].strip() + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + + return pieces + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%%d.g%%s" %% (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%%d.g%%s" %% (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%%d" %% pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%%d" %% pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%%s" %% pieces["short"] + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%%s" %% pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%%d" %% pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%%d-g%%s" %% (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%%s'" %% style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} + + +def get_versions(): + """Get version information or return default if unable to do so.""" + # I am in _version.py, which lives at ROOT/VERSIONFILE_SOURCE. If we have + # __file__, we can work backwards from there to the root. Some + # py2exe/bbfreeze/non-CPython implementations don't do __file__, in which + # case we can only use expanded keywords. + + cfg = get_config() + verbose = cfg.verbose + + try: + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, + verbose) + except NotThisMethod: + pass + + try: + root = os.path.realpath(__file__) + # versionfile_source is the relative path from the top of the source + # tree (where the .git directory might live) to this file. Invert + # this to find the root from __file__. + for i in cfg.versionfile_source.split('/'): + root = os.path.dirname(root) + except NameError: + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None} + + try: + pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) + return render(pieces, cfg.style) + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + return versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + except NotThisMethod: + pass + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", "date": None} +''' + + +@register_vcs_handler("git", "get_keywords") +def git_get_keywords(versionfile_abs): + """Extract version information from the given file.""" + # the code embedded in _version.py can just fetch the value of these + # keywords. When used from setup.py, we don't want to import _version.py, + # so we do it with a regexp instead. This function is not used from + # _version.py. + keywords = {} + try: + f = open(versionfile_abs, "r") + for line in f.readlines(): + if line.strip().startswith("git_refnames ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["refnames"] = mo.group(1) + if line.strip().startswith("git_full ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["full"] = mo.group(1) + if line.strip().startswith("git_date ="): + mo = re.search(r'=\s*"(.*)"', line) + if mo: + keywords["date"] = mo.group(1) + f.close() + except EnvironmentError: + pass + return keywords + + +@register_vcs_handler("git", "keywords") +def git_versions_from_keywords(keywords, tag_prefix, verbose): + """Get version information from git keywords.""" + if not keywords: + raise NotThisMethod("no keywords at all, weird") + date = keywords.get("date") + if date is not None: + # git-2.2.0 added "%cI", which expands to an ISO-8601 -compliant + # datestamp. However we prefer "%ci" (which expands to an "ISO-8601 + # -like" string, which we must then edit to make compliant), because + # it's been around since git-1.5.3, and it's too difficult to + # discover which version we're using, or to work around using an + # older one. + date = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + refnames = keywords["refnames"].strip() + if refnames.startswith("$Format"): + if verbose: + print("keywords are unexpanded, not using") + raise NotThisMethod("unexpanded keywords, not a git-archive tarball") + refs = set([r.strip() for r in refnames.strip("()").split(",")]) + # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of + # just "foo-1.0". If we see a "tag: " prefix, prefer those. + TAG = "tag: " + tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + if not tags: + # Either we're using git < 1.8.3, or there really are no tags. We use + # a heuristic: assume all version tags have a digit. The old git %d + # expansion behaves like git log --decorate=short and strips out the + # refs/heads/ and refs/tags/ prefixes that would let us distinguish + # between branches and tags. By ignoring refnames without digits, we + # filter out many common branch names like "release" and + # "stabilization", as well as "HEAD" and "master". + tags = set([r for r in refs if re.search(r'\d', r)]) + if verbose: + print("discarding '%s', no digits" % ",".join(refs - tags)) + if verbose: + print("likely tags: %s" % ",".join(sorted(tags))) + for ref in sorted(tags): + # sorting will prefer e.g. "2.0" over "2.0rc1" + if ref.startswith(tag_prefix): + r = ref[len(tag_prefix):] + if verbose: + print("picking %s" % r) + return {"version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": None, + "date": date} + # no suitable tags, so version is "0+unknown", but full hex is still there + if verbose: + print("no suitable tags, using unknown + full revision id") + return {"version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, "error": "no suitable tags", "date": None} + + +@register_vcs_handler("git", "pieces_from_vcs") +def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): + """Get version from 'git describe' in the root of the source tree. + + This only gets called if the git-archive 'subst' keywords were *not* + expanded, and _version.py hasn't already been rewritten with a short + version string, meaning we're inside a checked out source tree. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, + hide_stderr=True) + if rc != 0: + if verbose: + print("Directory %s not under git control" % root) + raise NotThisMethod("'git rev-parse --git-dir' returned error") + + # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] + # if there isn't one, this yields HEX[-dirty] (no NUM) + describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", + "--always", "--long", + "--match", "%s*" % tag_prefix], + cwd=root) + # --long was added in git-1.5.5 + if describe_out is None: + raise NotThisMethod("'git describe' failed") + describe_out = describe_out.strip() + full_out, rc = run_command(GITS, ["rev-parse", "HEAD"], cwd=root) + if full_out is None: + raise NotThisMethod("'git rev-parse' failed") + full_out = full_out.strip() + + pieces = {} + pieces["long"] = full_out + pieces["short"] = full_out[:7] # maybe improved later + pieces["error"] = None + + # parse describe_out. It will be like TAG-NUM-gHEX[-dirty] or HEX[-dirty] + # TAG might have hyphens. + git_describe = describe_out + + # look for -dirty suffix + dirty = git_describe.endswith("-dirty") + pieces["dirty"] = dirty + if dirty: + git_describe = git_describe[:git_describe.rindex("-dirty")] + + # now we have TAG-NUM-gHEX or HEX + + if "-" in git_describe: + # TAG-NUM-gHEX + mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + if not mo: + # unparseable. Maybe git-describe is misbehaving? + pieces["error"] = ("unable to parse git-describe output: '%s'" + % describe_out) + return pieces + + # tag + full_tag = mo.group(1) + if not full_tag.startswith(tag_prefix): + if verbose: + fmt = "tag '%s' doesn't start with prefix '%s'" + print(fmt % (full_tag, tag_prefix)) + pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" + % (full_tag, tag_prefix)) + return pieces + pieces["closest-tag"] = full_tag[len(tag_prefix):] + + # distance: number of commits since tag + pieces["distance"] = int(mo.group(2)) + + # commit: short hex revision ID + pieces["short"] = mo.group(3) + + else: + # HEX: no tags + pieces["closest-tag"] = None + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], + cwd=root) + pieces["distance"] = int(count_out) # total number of commits + + # commit date: see ISO-8601 comment in git_versions_from_keywords() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], + cwd=root)[0].strip() + pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) + + return pieces + + +def do_vcs_install(manifest_in, versionfile_source, ipy): + """Git-specific installation logic for Versioneer. + + For Git, this means creating/changing .gitattributes to mark _version.py + for export-subst keyword substitution. + """ + GITS = ["git"] + if sys.platform == "win32": + GITS = ["git.cmd", "git.exe"] + files = [manifest_in, versionfile_source] + if ipy: + files.append(ipy) + try: + me = __file__ + if me.endswith(".pyc") or me.endswith(".pyo"): + me = os.path.splitext(me)[0] + ".py" + versioneer_file = os.path.relpath(me) + except NameError: + versioneer_file = "versioneer.py" + files.append(versioneer_file) + present = False + try: + f = open(".gitattributes", "r") + for line in f.readlines(): + if line.strip().startswith(versionfile_source): + if "export-subst" in line.strip().split()[1:]: + present = True + f.close() + except EnvironmentError: + pass + if not present: + f = open(".gitattributes", "a+") + f.write("%s export-subst\n" % versionfile_source) + f.close() + files.append(".gitattributes") + run_command(GITS, ["add", "--"] + files) + + +def versions_from_parentdir(parentdir_prefix, root, verbose): + """Try to determine the version from the parent directory name. + + Source tarballs conventionally unpack into a directory that includes both + the project name and a version string. We will also support searching up + two directory levels for an appropriately named parent directory + """ + rootdirs = [] + + for i in range(3): + dirname = os.path.basename(root) + if dirname.startswith(parentdir_prefix): + return {"version": dirname[len(parentdir_prefix):], + "full-revisionid": None, + "dirty": False, "error": None, "date": None} + else: + rootdirs.append(root) + root = os.path.dirname(root) # up a level + + if verbose: + print("Tried directories %s but none started with prefix %s" % + (str(rootdirs), parentdir_prefix)) + raise NotThisMethod("rootdir doesn't start with parentdir_prefix") + + +SHORT_VERSION_PY = """ +# This file was generated by 'versioneer.py' (0.18) from +# revision-control system data, or from the parent directory name of an +# unpacked source archive. Distribution tarballs contain a pre-generated copy +# of this file. + +import json + +version_json = ''' +%s +''' # END VERSION_JSON + + +def get_versions(): + return json.loads(version_json) +""" + + +def versions_from_file(filename): + """Try to determine the version from _version.py if present.""" + try: + with open(filename) as f: + contents = f.read() + except EnvironmentError: + raise NotThisMethod("unable to read _version.py") + mo = re.search(r"version_json = '''\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) + if not mo: + mo = re.search(r"version_json = '''\r\n(.*)''' # END VERSION_JSON", + contents, re.M | re.S) + if not mo: + raise NotThisMethod("no version_json in _version.py") + return json.loads(mo.group(1)) + + +def write_to_version_file(filename, versions): + """Write the given version number to the given _version.py file.""" + os.unlink(filename) + contents = json.dumps(versions, sort_keys=True, + indent=1, separators=(",", ": ")) + with open(filename, "w") as f: + f.write(SHORT_VERSION_PY % contents) + + print("set %s to '%s'" % (filename, versions["version"])) + + +def plus_or_dot(pieces): + """Return a + if we don't already have one, else return a .""" + if "+" in pieces.get("closest-tag", ""): + return "." + return "+" + + +def render_pep440(pieces): + """Build up version string, with post-release "local version identifier". + + Our goal: TAG[+DISTANCE.gHEX[.dirty]] . Note that if you + get a tagged build and then dirty it, you'll get TAG+0.gHEX.dirty + + Exceptions: + 1: no tags. git_describe was just HEX. 0+untagged.DISTANCE.gHEX[.dirty] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += plus_or_dot(pieces) + rendered += "%d.g%s" % (pieces["distance"], pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + else: + # exception #1 + rendered = "0+untagged.%d.g%s" % (pieces["distance"], + pieces["short"]) + if pieces["dirty"]: + rendered += ".dirty" + return rendered + + +def render_pep440_pre(pieces): + """TAG[.post.devDISTANCE] -- No -dirty. + + Exceptions: + 1: no tags. 0.post.devDISTANCE + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += ".post.dev%d" % pieces["distance"] + else: + # exception #1 + rendered = "0.post.dev%d" % pieces["distance"] + return rendered + + +def render_pep440_post(pieces): + """TAG[.postDISTANCE[.dev0]+gHEX] . + + The ".dev0" means dirty. Note that .dev0 sorts backwards + (a dirty tree will appear "older" than the corresponding clean one), + but you shouldn't be releasing software with -dirty anyways. + + Exceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += plus_or_dot(pieces) + rendered += "g%s" % pieces["short"] + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + rendered += "+g%s" % pieces["short"] + return rendered + + +def render_pep440_old(pieces): + """TAG[.postDISTANCE[.dev0]] . + + The ".dev0" means dirty. + + Eexceptions: + 1: no tags. 0.postDISTANCE[.dev0] + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"] or pieces["dirty"]: + rendered += ".post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + else: + # exception #1 + rendered = "0.post%d" % pieces["distance"] + if pieces["dirty"]: + rendered += ".dev0" + return rendered + + +def render_git_describe(pieces): + """TAG[-DISTANCE-gHEX][-dirty]. + + Like 'git describe --tags --dirty --always'. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + if pieces["distance"]: + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render_git_describe_long(pieces): + """TAG-DISTANCE-gHEX[-dirty]. + + Like 'git describe --tags --dirty --always -long'. + The distance/hash is unconditional. + + Exceptions: + 1: no tags. HEX[-dirty] (note: no 'g' prefix) + """ + if pieces["closest-tag"]: + rendered = pieces["closest-tag"] + rendered += "-%d-g%s" % (pieces["distance"], pieces["short"]) + else: + # exception #1 + rendered = pieces["short"] + if pieces["dirty"]: + rendered += "-dirty" + return rendered + + +def render(pieces, style): + """Render the given version pieces into the requested style.""" + if pieces["error"]: + return {"version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None} + + if not style or style == "default": + style = "pep440" # the default + + if style == "pep440": + rendered = render_pep440(pieces) + elif style == "pep440-pre": + rendered = render_pep440_pre(pieces) + elif style == "pep440-post": + rendered = render_pep440_post(pieces) + elif style == "pep440-old": + rendered = render_pep440_old(pieces) + elif style == "git-describe": + rendered = render_git_describe(pieces) + elif style == "git-describe-long": + rendered = render_git_describe_long(pieces) + else: + raise ValueError("unknown style '%s'" % style) + + return {"version": rendered, "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], "error": None, + "date": pieces.get("date")} + + +class VersioneerBadRootError(Exception): + """The project root directory is unknown or missing key files.""" + + +def get_versions(verbose=False): + """Get the project version from whatever source is available. + + Returns dict with two keys: 'version' and 'full'. + """ + if "versioneer" in sys.modules: + # see the discussion in cmdclass.py:get_cmdclass() + del sys.modules["versioneer"] + + root = get_root() + cfg = get_config_from_root(root) + + assert cfg.VCS is not None, "please set [versioneer]VCS= in setup.cfg" + handlers = HANDLERS.get(cfg.VCS) + assert handlers, "unrecognized VCS '%s'" % cfg.VCS + verbose = verbose or cfg.verbose + assert cfg.versionfile_source is not None, \ + "please set versioneer.versionfile_source" + assert cfg.tag_prefix is not None, "please set versioneer.tag_prefix" + + versionfile_abs = os.path.join(root, cfg.versionfile_source) + + # extract version from first of: _version.py, VCS command (e.g. 'git + # describe'), parentdir. This is meant to work for developers using a + # source checkout, for users of a tarball created by 'setup.py sdist', + # and for users of a tarball/zipball created by 'git archive' or github's + # download-from-tag feature or the equivalent in other VCSes. + + get_keywords_f = handlers.get("get_keywords") + from_keywords_f = handlers.get("keywords") + if get_keywords_f and from_keywords_f: + try: + keywords = get_keywords_f(versionfile_abs) + ver = from_keywords_f(keywords, cfg.tag_prefix, verbose) + if verbose: + print("got version from expanded keyword %s" % ver) + return ver + except NotThisMethod: + pass + + try: + ver = versions_from_file(versionfile_abs) + if verbose: + print("got version from file %s %s" % (versionfile_abs, ver)) + return ver + except NotThisMethod: + pass + + from_vcs_f = handlers.get("pieces_from_vcs") + if from_vcs_f: + try: + pieces = from_vcs_f(cfg.tag_prefix, root, verbose) + ver = render(pieces, cfg.style) + if verbose: + print("got version from VCS %s" % ver) + return ver + except NotThisMethod: + pass + + try: + if cfg.parentdir_prefix: + ver = versions_from_parentdir(cfg.parentdir_prefix, root, verbose) + if verbose: + print("got version from parentdir %s" % ver) + return ver + except NotThisMethod: + pass + + if verbose: + print("unable to compute version") + + return {"version": "0+unknown", "full-revisionid": None, + "dirty": None, "error": "unable to compute version", + "date": None} + + +def get_version(): + """Get the short version string for this project.""" + return get_versions()["version"] + + +def get_cmdclass(): + """Get the custom setuptools/distutils subclasses used by Versioneer.""" + if "versioneer" in sys.modules: + del sys.modules["versioneer"] + # this fixes the "python setup.py develop" case (also 'install' and + # 'easy_install .'), in which subdependencies of the main project are + # built (using setup.py bdist_egg) in the same python process. Assume + # a main project A and a dependency B, which use different versions + # of Versioneer. A's setup.py imports A's Versioneer, leaving it in + # sys.modules by the time B's setup.py is executed, causing B to run + # with the wrong versioneer. Setuptools wraps the sub-dep builds in a + # sandbox that restores sys.modules to it's pre-build state, so the + # parent is protected against the child's "import versioneer". By + # removing ourselves from sys.modules here, before the child build + # happens, we protect the child from the parent's versioneer too. + # Also see https://github.com/warner/python-versioneer/issues/52 + + cmds = {} + + # we add "version" to both distutils and setuptools + from distutils.core import Command + + class cmd_version(Command): + description = "report generated version string" + user_options = [] + boolean_options = [] + + def initialize_options(self): + pass + + def finalize_options(self): + pass + + def run(self): + vers = get_versions(verbose=True) + print("Version: %s" % vers["version"]) + print(" full-revisionid: %s" % vers.get("full-revisionid")) + print(" dirty: %s" % vers.get("dirty")) + print(" date: %s" % vers.get("date")) + if vers["error"]: + print(" error: %s" % vers["error"]) + cmds["version"] = cmd_version + + # we override "build_py" in both distutils and setuptools + # + # most invocation pathways end up running build_py: + # distutils/build -> build_py + # distutils/install -> distutils/build ->.. + # setuptools/bdist_wheel -> distutils/install ->.. + # setuptools/bdist_egg -> distutils/install_lib -> build_py + # setuptools/install -> bdist_egg ->.. + # setuptools/develop -> ? + # pip install: + # copies source tree to a tempdir before running egg_info/etc + # if .git isn't copied too, 'git describe' will fail + # then does setup.py bdist_wheel, or sometimes setup.py install + # setup.py egg_info -> ? + + # we override different "build_py" commands for both environments + if "setuptools" in sys.modules: + from setuptools.command.build_py import build_py as _build_py + else: + from distutils.command.build_py import build_py as _build_py + + class cmd_build_py(_build_py): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + _build_py.run(self) + # now locate _version.py in the new build/ directory and replace + # it with an updated value + if cfg.versionfile_build: + target_versionfile = os.path.join(self.build_lib, + cfg.versionfile_build) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + cmds["build_py"] = cmd_build_py + + if "cx_Freeze" in sys.modules: # cx_freeze enabled? + from cx_Freeze.dist import build_exe as _build_exe + # nczeczulin reports that py2exe won't like the pep440-style string + # as FILEVERSION, but it can be used for PRODUCTVERSION, e.g. + # setup(console=[{ + # "version": versioneer.get_version().split("+", 1)[0], # FILEVERSION + # "product_version": versioneer.get_version(), + # ... + + class cmd_build_exe(_build_exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _build_exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + cmds["build_exe"] = cmd_build_exe + del cmds["build_py"] + + if 'py2exe' in sys.modules: # py2exe enabled? + try: + from py2exe.distutils_buildexe import py2exe as _py2exe # py3 + except ImportError: + from py2exe.build_exe import py2exe as _py2exe # py2 + + class cmd_py2exe(_py2exe): + def run(self): + root = get_root() + cfg = get_config_from_root(root) + versions = get_versions() + target_versionfile = cfg.versionfile_source + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, versions) + + _py2exe.run(self) + os.unlink(target_versionfile) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % + {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + cmds["py2exe"] = cmd_py2exe + + # we override different "sdist" commands for both environments + if "setuptools" in sys.modules: + from setuptools.command.sdist import sdist as _sdist + else: + from distutils.command.sdist import sdist as _sdist + + class cmd_sdist(_sdist): + def run(self): + versions = get_versions() + self._versioneer_generated_versions = versions + # unless we update this, the command will keep using the old + # version + self.distribution.metadata.version = versions["version"] + return _sdist.run(self) + + def make_release_tree(self, base_dir, files): + root = get_root() + cfg = get_config_from_root(root) + _sdist.make_release_tree(self, base_dir, files) + # now locate _version.py in the new base_dir directory + # (remembering that it may be a hardlink) and replace it with an + # updated value + target_versionfile = os.path.join(base_dir, cfg.versionfile_source) + print("UPDATING %s" % target_versionfile) + write_to_version_file(target_versionfile, + self._versioneer_generated_versions) + cmds["sdist"] = cmd_sdist + + return cmds + + +CONFIG_ERROR = """ +setup.cfg is missing the necessary Versioneer configuration. You need +a section like: + + [versioneer] + VCS = git + style = pep440 + versionfile_source = src/myproject/_version.py + versionfile_build = myproject/_version.py + tag_prefix = + parentdir_prefix = myproject- + +You will also need to edit your setup.py to use the results: + + import versioneer + setup(version=versioneer.get_version(), + cmdclass=versioneer.get_cmdclass(), ...) + +Please read the docstring in ./versioneer.py for configuration instructions, +edit setup.cfg, and re-run the installer or 'python versioneer.py setup'. +""" + +SAMPLE_CONFIG = """ +# See the docstring in versioneer.py for instructions. Note that you must +# re-run 'versioneer.py setup' after changing this section, and commit the +# resulting files. + +[versioneer] +#VCS = git +#style = pep440 +#versionfile_source = +#versionfile_build = +#tag_prefix = +#parentdir_prefix = + +""" + +INIT_PY_SNIPPET = """ +from ._version import get_versions +__version__ = get_versions()['version'] +del get_versions +""" + + +def do_setup(): + """Main VCS-independent setup function for installing Versioneer.""" + root = get_root() + try: + cfg = get_config_from_root(root) + except (EnvironmentError, configparser.NoSectionError, + configparser.NoOptionError) as e: + if isinstance(e, (EnvironmentError, configparser.NoSectionError)): + print("Adding sample versioneer config to setup.cfg", + file=sys.stderr) + with open(os.path.join(root, "setup.cfg"), "a") as f: + f.write(SAMPLE_CONFIG) + print(CONFIG_ERROR, file=sys.stderr) + return 1 + + print(" creating %s" % cfg.versionfile_source) + with open(cfg.versionfile_source, "w") as f: + LONG = LONG_VERSION_PY[cfg.VCS] + f.write(LONG % {"DOLLAR": "$", + "STYLE": cfg.style, + "TAG_PREFIX": cfg.tag_prefix, + "PARENTDIR_PREFIX": cfg.parentdir_prefix, + "VERSIONFILE_SOURCE": cfg.versionfile_source, + }) + + ipy = os.path.join(os.path.dirname(cfg.versionfile_source), + "__init__.py") + if os.path.exists(ipy): + try: + with open(ipy, "r") as f: + old = f.read() + except EnvironmentError: + old = "" + if INIT_PY_SNIPPET not in old: + print(" appending to %s" % ipy) + with open(ipy, "a") as f: + f.write(INIT_PY_SNIPPET) + else: + print(" %s unmodified" % ipy) + else: + print(" %s doesn't exist, ok" % ipy) + ipy = None + + # Make sure both the top-level "versioneer.py" and versionfile_source + # (PKG/_version.py, used by runtime code) are in MANIFEST.in, so + # they'll be copied into source distributions. Pip won't be able to + # install the package without this. + manifest_in = os.path.join(root, "MANIFEST.in") + simple_includes = set() + try: + with open(manifest_in, "r") as f: + for line in f: + if line.startswith("include "): + for include in line.split()[1:]: + simple_includes.add(include) + except EnvironmentError: + pass + # That doesn't cover everything MANIFEST.in can do + # (http://docs.python.org/2/distutils/sourcedist.html#commands), so + # it might give some false negatives. Appending redundant 'include' + # lines is safe, though. + if "versioneer.py" not in simple_includes: + print(" appending 'versioneer.py' to MANIFEST.in") + with open(manifest_in, "a") as f: + f.write("include versioneer.py\n") + else: + print(" 'versioneer.py' already in MANIFEST.in") + if cfg.versionfile_source not in simple_includes: + print(" appending versionfile_source ('%s') to MANIFEST.in" % + cfg.versionfile_source) + with open(manifest_in, "a") as f: + f.write("include %s\n" % cfg.versionfile_source) + else: + print(" versionfile_source already in MANIFEST.in") + + # Make VCS-specific changes. For git, this means creating/changing + # .gitattributes to mark _version.py for export-subst keyword + # substitution. + do_vcs_install(manifest_in, cfg.versionfile_source, ipy) + return 0 + + +def scan_setup_py(): + """Validate the contents of setup.py against Versioneer's expectations.""" + found = set() + setters = False + errors = 0 + with open("setup.py", "r") as f: + for line in f.readlines(): + if "import versioneer" in line: + found.add("import") + if "versioneer.get_cmdclass()" in line: + found.add("cmdclass") + if "versioneer.get_version()" in line: + found.add("get_version") + if "versioneer.VCS" in line: + setters = True + if "versioneer.versionfile_source" in line: + setters = True + if len(found) != 3: + print("") + print("Your setup.py appears to be missing some important items") + print("(but I might be wrong). Please make sure it has something") + print("roughly like the following:") + print("") + print(" import versioneer") + print(" setup( version=versioneer.get_version(),") + print(" cmdclass=versioneer.get_cmdclass(), ...)") + print("") + errors += 1 + if setters: + print("You should remove lines like 'versioneer.VCS = ' and") + print("'versioneer.versionfile_source = ' . This configuration") + print("now lives in setup.cfg, and should be removed from setup.py") + print("") + errors += 1 + return errors + + +if __name__ == "__main__": + cmd = sys.argv[1] + if cmd == "setup": + errors = do_setup() + errors += scan_setup_py() + if errors: + sys.exit(1)