diff --git a/.gitignore b/.gitignore index ebd4782f2..853591131 100644 --- a/.gitignore +++ b/.gitignore @@ -26,4 +26,5 @@ dist/ examples/searchcommands_app/package/default/commands.conf examples/searchcommands_app/package/bin/packages tests/searchcommands/apps/app_with_logging_configuration/*.log -*.observed \ No newline at end of file +*.observed +venv/ \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 37640ff1e..d8d5f77b3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -30,8 +30,8 @@ before_install: - mkdir -p $SPLUNK_HOME/var/log/splunk env: - - SPLUNK_VERSION=6.6-sdk - SPLUNK_VERSION=7.0-sdk + - SPLUNK_VERSION=7.2-sdk language: python diff --git a/CHANGELOG.md b/CHANGELOG.md index 5668b6cdf..a603e3808 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Splunk SDK for Python Changelog +## Versiom 1.6.6 + +### Bug fixes + +* Fix ssl verify to require certs when true + +### Minor changes + +* Make the explorer example compatible w/ Python 3 +* Add full support for unicode in SearchCommands +* Add return code for invalid_args block + ## Version 1.6.5 ### Bug fixes diff --git a/README.md b/README.md index 751569d3b..4e67795ff 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ [![Build Status](https://travis-ci.org/splunk/splunk-sdk-python.svg?branch=master)](https://travis-ci.org/splunk/splunk-sdk-python) # The Splunk Software Development Kit for Python -#### Version 1.6.5 +#### Version 1.6.6 The Splunk Software Development Kit (SDK) for Python contains library code and examples designed to enable developers to build applications using Splunk. diff --git a/docs/conf.py b/docs/conf.py index c68aa66b6..4755a4b89 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -248,3 +248,5 @@ # How to display URL addresses: 'footnote', 'no', or 'inline'. #texinfo_show_urls = 'footnote' + +autoclass_content = 'both' diff --git a/examples/explorer/explorer.py b/examples/explorer/explorer.py index 60a9aeff3..be3dc3279 100755 --- a/examples/explorer/explorer.py +++ b/examples/explorer/explorer.py @@ -27,7 +27,8 @@ except ImportError: raise Exception("Add the SDK repository to your PYTHONPATH to run the examples " "(e.g., export PYTHONPATH=~/splunk-sdk-python.") -import urllib + +from splunklib.six.moves import urllib PORT = 8080 @@ -57,7 +58,7 @@ def main(argv): args.append(('owner', opts.kwargs['owner'])) # Encode these arguments - args = urllib.urlencode(args) + args = urllib.parse.urlencode(args) # Launch the browser webbrowser.open("file://%s" % os.path.join(os.getcwd(), "explorer.html?%s" % args)) diff --git a/examples/explorer/server.py b/examples/explorer/server.py index 5f9caaf63..9626737c3 100755 --- a/examples/explorer/server.py +++ b/examples/explorer/server.py @@ -16,16 +16,18 @@ from __future__ import absolute_import from __future__ import print_function -import splunklib.six.moves.SimpleHTTPServer -import splunklib.six.moves.socketserver -import urllib2 import sys -import StringIO -from splunklib import six +import os + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..")) + +from splunklib.six import iteritems +from splunklib.six.moves import socketserver, SimpleHTTPServer, StringIO, urllib PORT = 8080 -class RedirectHandler(six.moves.SimpleHTTPServer.SimpleHTTPRequestHandler): + +class RedirectHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): def do_GET(self): redirect_url, headers = self.get_url_and_headers() if redirect_url is None: @@ -83,13 +85,13 @@ def make_request(self, url, method, data, headers): try: # Make the request - request = urllib2.Request(url, data, headers) + request = urllib.Request(url, data, headers) request.get_method = lambda: method - response = urllib2.urlopen(request) + response = urllib.urlopen(request) # We were successful, so send the response code self.send_response(response.code, message=response.msg) - for key, value in six.iteritems(dict(response.headers)): + for key, value in iteritems(dict(response.headers)): # Optionally log the headers #self.log_message("%s: %s" % (key, value)) @@ -105,16 +107,16 @@ def make_request(self, url, method, data, headers): # Copy the response to the output self.copyfile(response, self.wfile) - except urllib2.HTTPError as e: + except urllib.HTTPError as e: # On errors, log the response code and message self.log_message("Code: %s (%s)", e.code, e.msg) - for key, value in six.iteritems(dict(e.hdrs)): + for key, value in iteritems(dict(e.hdrs)): # On errors, we always log the headers self.log_message("%s: %s", key, value) response_text = e.fp.read() - response_file = StringIO.StringIO(response_text) + response_file = StringIO(response_text) # On errors, we also log the response text self.log_message("Response: %s", response_text) @@ -135,10 +137,10 @@ def make_request(self, url, method, data, headers): # Finally, send the error itself self.copyfile(response_file, self.wfile) -class ReuseableSocketTCPServer(six.moves.socketserver.TCPServer): +class ReuseableSocketTCPServer(socketserver.TCPServer): def __init__(self, *args, **kwargs): self.allow_reuse_address = True - six.moves.socketserver.TCPServer.__init__(self, *args, **kwargs) + socketserver.TCPServer.__init__(self, *args, **kwargs) def serve(port = PORT): Handler = RedirectHandler diff --git a/examples/searchcommands_app/setup.py b/examples/searchcommands_app/setup.py index 48e12e628..ab8d6cac4 100755 --- a/examples/searchcommands_app/setup.py +++ b/examples/searchcommands_app/setup.py @@ -439,7 +439,7 @@ def run(self): setup( description='Custom Search Command examples', name=os.path.basename(project_dir), - version='1.6.5', + version='1.6.6', author='Splunk, Inc.', author_email='devinfo@splunk.com', url='http://github.com/splunk/splunk-sdk-python', diff --git a/splunklib/__init__.py b/splunklib/__init__.py index e7a90ddcc..59daf9ea8 100644 --- a/splunklib/__init__.py +++ b/splunklib/__init__.py @@ -16,5 +16,5 @@ from __future__ import absolute_import from splunklib.six.moves import map -__version_info__ = (1, 6, 5) +__version_info__ = (1, 6, 6) __version__ = ".".join(map(str, __version_info__)) diff --git a/splunklib/binding.py b/splunklib/binding.py index 74d917377..3fe7c8495 100644 --- a/splunklib/binding.py +++ b/splunklib/binding.py @@ -25,30 +25,30 @@ """ from __future__ import absolute_import + +import io import logging import socket import ssl -from io import BytesIO - -from splunklib.six.moves import urllib -import io import sys - from base64 import b64encode +from contextlib import contextmanager from datetime import datetime from functools import wraps +from io import BytesIO +from xml.etree.ElementTree import XML + +from splunklib import six from splunklib.six import StringIO +from splunklib.six.moves import urllib -from contextlib import contextmanager +from .data import record -from xml.etree.ElementTree import XML -from splunklib import six try: from xml.etree.ElementTree import ParseError except ImportError as e: from xml.parsers.expat import ExpatError as ParseError -from .data import record __all__ = [ "AuthenticationError", @@ -449,6 +449,8 @@ class Context(object): :type username: ``string`` :param password: The password for the Splunk account. :type password: ``string`` + :param headers: List of extra HTTP headers to send (optional). + :type headers: ``list`` of 2-tuples. :param handler: The HTTP request handler (optional). :returns: A ``Context`` instance. @@ -465,7 +467,8 @@ class Context(object): c = binding.Context(cookie="splunkd_8089=...") """ def __init__(self, handler=None, **kwargs): - self.http = HttpLib(handler, kwargs.get("verify", True)) + self.http = HttpLib(handler, kwargs.get("verify", False), key_file=kwargs.get("key_file"), + cert_file=kwargs.get("cert_file")) # Default to False for backward compat self.token = kwargs.get("token", _NoAuthenticationToken) if self.token is None: # In case someone explicitly passes token=None self.token = _NoAuthenticationToken @@ -478,6 +481,7 @@ def __init__(self, handler=None, **kwargs): self.password = kwargs.get("password", "") self.basic = kwargs.get("basic", False) self.autologin = kwargs.get("autologin", False) + self.additional_headers = kwargs.get("headers", []) # Store any cookies in the self.http._cookies dict if "cookie" in kwargs and kwargs['cookie'] not in [None, _NoAuthenticationToken]: @@ -613,7 +617,7 @@ def delete(self, path_segment, owner=None, app=None, sharing=None, **query): @_authentication @_log_duration - def get(self, path_segment, owner=None, app=None, sharing=None, **query): + def get(self, path_segment, owner=None, app=None, headers=None, sharing=None, **query): """Performs a GET operation from the REST path segment with the given namespace and query. @@ -636,6 +640,8 @@ def get(self, path_segment, owner=None, app=None, sharing=None, **query): :type owner: ``string`` :param app: The app context of the namespace (optional). :type app: ``string`` + :param headers: List of extra HTTP headers to send (optional). + :type headers: ``list`` of 2-tuples. :param sharing: The sharing mode of the namespace (optional). :type sharing: ``string`` :param query: All other keyword arguments, which are used as query @@ -663,10 +669,14 @@ def get(self, path_segment, owner=None, app=None, sharing=None, **query): c.logout() c.get('apps/local') # raises AuthenticationError """ + if headers is None: + headers = [] + path = self.authority + self._abspath(path_segment, owner=owner, app=app, sharing=sharing) logging.debug("GET request to %s (body: %s)", path, repr(query)) - response = self.http.get(path, self._auth_headers, **query) + all_headers = headers + self.additional_headers + self._auth_headers + response = self.http.get(path, all_headers, **query) return response @_authentication @@ -738,7 +748,7 @@ def post(self, path_segment, owner=None, app=None, sharing=None, headers=None, * path = self.authority + self._abspath(path_segment, owner=owner, app=app, sharing=sharing) logging.debug("POST request to %s (body: %s)", path, repr(query)) - all_headers = headers + self._auth_headers + all_headers = headers + self.additional_headers + self._auth_headers response = self.http.post(path, all_headers, **query) return response @@ -804,7 +814,7 @@ def request(self, path_segment, method="GET", headers=None, body="", path = self.authority \ + self._abspath(path_segment, owner=owner, app=app, sharing=sharing) - all_headers = headers + self._auth_headers + all_headers = headers + self.additional_headers + self._auth_headers logging.debug("%s request to %s (headers: %s, body: %s)", method, path, str(all_headers), repr(body)) response = self.http.request(path, @@ -858,6 +868,7 @@ def login(self): self.authority + self._abspath("/services/auth/login"), username=self.username, password=self.password, + headers=self.additional_headers, cookie="1") # In Splunk 6.2+, passing "cookie=1" will return the "set-cookie" header body = response.body.read() @@ -968,6 +979,8 @@ def connect(**kwargs): :type username: ``string`` :param password: The password for the Splunk account. :type password: ``string`` + :param headers: List of extra HTTP headers to send (optional). + :type headers: ``list`` of 2-tuples. :param autologin: When ``True``, automatically tries to log in again if the session terminates. :type autologin: ``Boolean`` @@ -1108,8 +1121,11 @@ class HttpLib(object): If using the default handler, SSL verification can be disabled by passing verify=False. """ - def __init__(self, custom_handler=None, verify=True): - self.handler = handler(verify=verify) if custom_handler is None else custom_handler + def __init__(self, custom_handler=None, verify=False, key_file=None, cert_file=None): + if custom_handler is None: + self.handler = handler(verify=verify, key_file=key_file, cert_file=cert_file) + else: + self.handler = custom_handler self._cookies = {} def delete(self, url, headers=None, **kwargs): @@ -1190,7 +1206,7 @@ def post(self, url, headers=None, **kwargs): # to support the receivers/stream endpoint. if 'body' in kwargs: # We only use application/x-www-form-urlencoded if there is no other - # Content-Type header present. This can happen in cases where we + # Content-Type header present. This can happen in cases where we # send requests as application/json, e.g. for KV Store. if len([x for x in headers if x[0].lower() == "content-type"]) == 0: headers.append(("Content-Type", "application/x-www-form-urlencoded")) @@ -1280,8 +1296,8 @@ def peek(self, size): def close(self): """Closes this response.""" - if _connection: - _connection.close() + if self._connection: + self._connection.close() self._response.close() def read(self, size = None): @@ -1317,7 +1333,7 @@ def readinto(self, byte_array): return bytes_read -def handler(key_file=None, cert_file=None, timeout=None, verify=True): +def handler(key_file=None, cert_file=None, timeout=None, verify=False): """This class returns an instance of the default HTTP request handler using the values you provide. @@ -1341,7 +1357,7 @@ def connect(scheme, host, port): if cert_file is not None: kwargs['cert_file'] = cert_file # If running Python 2.7.9+, disable SSL certificate validation - if (sys.version_info >= (2,7,9) and key_file is None and cert_file is None) or not verify: + if (sys.version_info >= (2,7,9) and key_file is None and cert_file is None) and not verify: kwargs['context'] = ssl._create_unverified_context() return six.moves.http_client.HTTPSConnection(host, port, **kwargs) raise ValueError("unsupported scheme: %s" % scheme) @@ -1352,7 +1368,7 @@ def request(url, message, **kwargs): head = { "Content-Length": str(len(body)), "Host": host, - "User-Agent": "splunk-sdk-python/1.6.5", + "User-Agent": "splunk-sdk-python/1.6.6", "Accept": "*/*", "Connection": "Close", } # defaults diff --git a/splunklib/client.py b/splunklib/client.py index 4664c0d01..1e624ba42 100644 --- a/splunklib/client.py +++ b/splunklib/client.py @@ -58,19 +58,22 @@ my_app.package() # Creates a compressed package of this application """ +import contextlib import datetime import json -from splunklib.six.moves import urllib import logging -from time import sleep -from datetime import datetime, timedelta import socket -import contextlib +from datetime import datetime, timedelta +from time import sleep from splunklib import six -from .binding import Context, HTTPError, AuthenticationError, namespace, UrlEncoded, _encode, _make_cookie_header, _NoAuthenticationToken -from .data import record +from splunklib.six.moves import urllib + from . import data +from .binding import (AuthenticationError, Context, HTTPError, UrlEncoded, + _encode, _make_cookie_header, _NoAuthenticationToken, + namespace) +from .data import record __all__ = [ "connect", @@ -193,8 +196,11 @@ def _path(base, name): # Load an atom record from the body of the given response +# this will ultimately be sent to an xml ElementTree so we +# should use the xmlcharrefreplace option def _load_atom(response, match=None): - return data.load(response.body.read().decode('utf-8'), match) + return data.load(response.body.read() + .decode('utf-8', 'xmlcharrefreplace'), match) # Load an array of atom entries from the body of the given response @@ -557,7 +563,7 @@ def restart(self, timeout=None): # This message will be deleted once the server actually restarts. self.messages.create(name="restart_required", **msg) result = self.post("/services/server/control/restart") - if timeout is None: + if timeout is None: return result start = datetime.now() diff = timedelta(seconds=timeout) @@ -1631,7 +1637,7 @@ def get(self, name="", owner=None, app=None, sharing=None, **query): and ``status`` Example: - + import splunklib.client s = client.service(...) saved_searches = s.saved_searches @@ -1685,7 +1691,7 @@ def __getitem__(self, key): # The superclass implementation is designed for collections that contain # entities. This collection (Configurations) contains collections # (ConfigurationFile). - # + # # The configurations endpoint returns multiple entities when we ask for a single file. # This screws up the default implementation of __getitem__ from Collection, which thinks # that multiple entities means a name collision, so we have to override it here. @@ -1749,9 +1755,9 @@ class Stanza(Entity): """This class contains a single configuration stanza.""" def submit(self, stanza): - """Adds keys to the current configuration stanza as a + """Adds keys to the current configuration stanza as a dictionary of key-value pairs. - + :param stanza: A dictionary of key-value pairs for the stanza. :type stanza: ``dict`` :return: The :class:`Stanza` object. @@ -1962,7 +1968,7 @@ def attach(self, host=None, source=None, sourcetype=None): cookie_or_auth_header.encode('utf-8'), b"X-Splunk-Input-Mode: Streaming\r\n", b"\r\n"] - + for h in headers: sock.write(h) return sock @@ -3695,13 +3701,13 @@ def batch_find(self, *dbqueries): :param dbqueries: Array of individual queries as dictionaries :type dbqueries: ``array`` of ``dict`` - + :return: Results of each query :rtype: ``array`` of ``array`` """ - if len(dbqueries) < 1: + if len(dbqueries) < 1: raise Exception('Must have at least one query.') - + data = json.dumps(dbqueries) return json.loads(self._post('batch_find', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) @@ -3712,13 +3718,13 @@ def batch_save(self, *documents): :param documents: Array of documents to save as dictionaries :type documents: ``array`` of ``dict`` - + :return: Results of update operation as overall stats :rtype: ``dict`` """ - if len(documents) < 1: + if len(documents) < 1: raise Exception('Must have at least one document.') - + data = json.dumps(documents) return json.loads(self._post('batch_save', headers=KVStoreCollectionData.JSON_HEADER, body=data).body.read().decode('utf-8')) diff --git a/splunklib/modularinput/script.py b/splunklib/modularinput/script.py index f9af0f887..040a07d9f 100755 --- a/splunklib/modularinput/script.py +++ b/splunklib/modularinput/script.py @@ -102,6 +102,7 @@ def run_script(self, args, event_writer, input_stream): err_string = "ERROR Invalid arguments to modular input script:" + ' '.join( args) event_writer._err.write(err_string) + return 1 except Exception as e: err_string = EventWriter.ERROR + str(e) diff --git a/splunklib/searchcommands/internals.py b/splunklib/searchcommands/internals.py index 78e643b0d..02634c081 100644 --- a/splunklib/searchcommands/internals.py +++ b/splunklib/searchcommands/internals.py @@ -22,7 +22,7 @@ from collections import OrderedDict # must be python 2.7 except ImportError: from ..ordereddict import OrderedDict -from splunklib.six.moves import cStringIO as StringIO +from splunklib.six.moves import StringIO from itertools import chain from splunklib.six.moves import map as imap from json import JSONDecoder, JSONEncoder diff --git a/splunklib/searchcommands/search_command.py b/splunklib/searchcommands/search_command.py index f3581026e..47918abb6 100644 --- a/splunklib/searchcommands/search_command.py +++ b/splunklib/searchcommands/search_command.py @@ -27,7 +27,7 @@ except ImportError: from ..ordereddict import OrderedDict from copy import deepcopy -from splunklib.six.moves import cStringIO as StringIO +from splunklib.six.moves import StringIO from itertools import chain, islice from splunklib.six.moves import filter as ifilter, map as imap, zip as izip from splunklib import six @@ -850,7 +850,6 @@ def _execute(self, ifile, process): @staticmethod def _read_chunk(ifile): - # noinspection PyBroadException try: header = ifile.readline() diff --git a/splunklib/searchcommands/validators.py b/splunklib/searchcommands/validators.py index eb1dc5cf0..66329375c 100644 --- a/splunklib/searchcommands/validators.py +++ b/splunklib/searchcommands/validators.py @@ -18,7 +18,7 @@ from json.encoder import encode_basestring_ascii as json_encode_string from collections import namedtuple -from splunklib.six.moves import cStringIO as StringIO +from splunklib.six.moves import StringIO from io import open import csv import os diff --git a/tests/searchcommands/test_search_command.py b/tests/searchcommands/test_search_command.py index 1e89e9ae1..0f0d71c13 100755 --- a/tests/searchcommands/test_search_command.py +++ b/tests/searchcommands/test_search_command.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +# coding=utf-8 # # Copyright 2011-2015 Splunk, Inc. # @@ -22,7 +23,7 @@ from splunklib.searchcommands.search_command import SearchCommand from splunklib.client import Service -from splunklib.six.moves import cStringIO as StringIO +from splunklib.six.moves import StringIO from splunklib.six.moves import zip as izip from json.encoder import encode_basestring as encode_string from unittest import main, TestCase @@ -388,7 +389,7 @@ def test_process_scpv2(self): '"required_option_1=value_1",' '"required_option_2=value_2"' '],' - '"search": "%7C%20inputlookup%20tweets%20%7C%20countmatches%20fieldname%3Dword_count%20pattern%3D%22%5Cw%2B%22%20text%20record%3Dt%20%7C%20export%20add_timestamp%3Df%20add_offset%3Dt%20format%3Dcsv%20segmentation%3Draw",' + '"search": "Ａ%7C%20inputlookup%20tweets%20%7C%20countmatches%20fieldname%3Dword_count%20pattern%3D%22%5Cw%2B%22%20text%20record%3Dt%20%7C%20export%20add_timestamp%3Df%20add_offset%3Dt%20format%3Dcsv%20segmentation%3Draw",' '"earliest_time": "0",' '"session_key": "0JbG1fJEvXrL6iYZw9y7tmvd6nHjTKj7ggaE7a4Jv5R0UIbeYJ65kThn^3hiNeoqzMT_LOtLpVR3Y8TIJyr5bkHUElMijYZ8l14wU0L4n^Oa5QxepsZNUIIQCBm^",' '"owner": "admin",' @@ -492,7 +493,7 @@ def test_process_scpv2(self): self.assertEqual(command_metadata.searchinfo.latest_time, 0.0) self.assertEqual(command_metadata.searchinfo.owner, 'admin') self.assertEqual(command_metadata.searchinfo.raw_args, command_metadata.searchinfo.args) - self.assertEqual(command_metadata.searchinfo.search, '| inputlookup tweets | countmatches fieldname=word_count pattern="\\w+" text record=t | export add_timestamp=f add_offset=t format=csv segmentation=raw') + self.assertEqual(command_metadata.searchinfo.search, 'Ａ| inputlookup tweets | countmatches fieldname=word_count pattern="\\w+" text record=t | export add_timestamp=f add_offset=t format=csv segmentation=raw') self.assertEqual(command_metadata.searchinfo.session_key, '0JbG1fJEvXrL6iYZw9y7tmvd6nHjTKj7ggaE7a4Jv5R0UIbeYJ65kThn^3hiNeoqzMT_LOtLpVR3Y8TIJyr5bkHUElMijYZ8l14wU0L4n^Oa5QxepsZNUIIQCBm^') self.assertEqual(command_metadata.searchinfo.sid, '1433261372.158') self.assertEqual(command_metadata.searchinfo.splunk_version, '20150522') diff --git a/tests/test_service.py b/tests/test_service.py index 1ae1f59af..56b88c061 100755 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -113,7 +113,7 @@ def test_parse_fail(self): def test_restart(self): service = client.connect(**self.opts.kwargs) - self.service.restart(timeout=120) + self.service.restart(timeout=300) service.login() # Make sure we are awake def test_read_outputs_with_type(self):