Skip to content

Commit

Permalink
dump, commit: Test for both non-empty and empty transaction extensions
Browse files Browse the repository at this point in the history
Currently we exercise zodbdump and zodbcommit+zodbdump with non-empty
extensions, which works if ZODB is patched for txn.extension_bytes
support, but fails on pristine ZODB.

Support for txn.extension_bytes cannot get into upstream ZODB for
more than a year:

zopefoundation/ZODB#183
zopefoundation/ZODB#207

and  even if it somehow will make it, it will likely be only in ZODB5,
while we still care to support ZODB4 and ZODB3.

Skipping zodbdump / zodbcommit tests, if a ZODB does not have
txn.extension_bytes support, would result in significant reduction of
zodbtools test coverage, because practically that is the current
situation with all upstream ZODB{3,4,5}. Dropping test coverage for
non-empty extensions is neither a good option.

For those reason, let's rework the tests and test both zodbdump and
zodbcommit with two scenarios:

1. on a test database where transactions extensions are always empty.
   This should work on all ZODB irregardless of whether
   txn.extension_bytes patch is there or not.

2. on a test database where transactions extensions are present.
   This should work if ZODB has txn.extension_bytes support, but if not,
   we can mark this case as xfail, since the failure is expected.

This way we make the testsuite pass irregardless of whether
txn.extension_bytes support is there, and we don't abandon dump/commit
testing coverage.

/helped-by Jérome Perrin <[email protected]>
  • Loading branch information
navytux committed Jan 10, 2019
1 parent 8c76eae commit 425e665
Show file tree
Hide file tree
Showing 8 changed files with 160 additions and 17 deletions.
56 changes: 56 additions & 0 deletions zodbtools/test/conftest.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <[email protected]>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.

import pytest
from zodbtools.test.testutil import zext_supported

# zext is a test fixture function object that allows to exercise 2 cases:
#
# - when ZODB does not have txn.extension_bytes support
# - when ZODB might have txn.extension_bytes support
#
# in a test, zext should be used as as follows:
#
# def test_something(zext):
# # bytes for an extension dict
# raw_ext = dumps({...})
#
# # will be either same as raw_ext, or b'' if ZODB lacks txn.extension_bytes support
# raw_ext = zext(raw_ext)
#
# # zext.disabled indicates whether testing for non-empty extension was disabled.
# if zext.disabled:
# ...
@pytest.fixture(params=['!zext', 'zext'])
def zext(request):
if request.param == '!zext':
# txn.extension_bytes is not working - always test with empty extension
def _(ext):
return b''
_.disabled = True
return _
else:
# txn.extension_bytes might be working - test with given extension and
# xfail if ZODB does not have necessary support.
def _(ext):
return ext
_.disabled = False
if not zext_supported():
request.applymarker(pytest.mark.xfail(reason='ZODB does not have txn.extension_bytes support'))
return _
42 changes: 32 additions & 10 deletions zodbtools/test/gen_testdata.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2017 Nexedi SA and Contributors.
# Kirill Smelkov <[email protected]>
# Copyright (C) 2017-2019 Nexedi SA and Contributors.
# Kirill Smelkov <[email protected]>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
Expand Down Expand Up @@ -60,7 +60,12 @@ def hex64(packed):
return '0x%016x' % unpack64(packed)

# make time.time() predictable
_xtime = time.mktime(time.strptime("04 Jan 1979", "%d %b %Y"))
_xtime0 = time.mktime(time.strptime("04 Jan 1979", "%d %b %Y"))
def xtime_reset():
global _xtime
_xtime = _xtime0
xtime_reset()

def xtime():
global _xtime
_xtime += 1.1
Expand Down Expand Up @@ -94,7 +99,7 @@ def __setstate__(self, state):

# prepare extension dictionary for subject
alnum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
def ext(subj):
def ext4subj(subj):
d = {"x-generator": "zodb/py%s (%s)" % (sys.version_info.major, subj)}

# also add some random 'x-cookie'
Expand All @@ -115,8 +120,16 @@ def ext(subj):

return ext

# gen_testdb generates test FileStorage database @ outfs_path
def gen_testdb(outfs_path):
# gen_testdb generates test FileStorage database @ outfs_path.
#
# zext indicates whether or not to include non-empty extension into transactions.
def gen_testdb(outfs_path, zext=True):
xtime_reset()

ext = ext4subj
if not zext:
def ext(subj): return {}

logging.basicConfig()

# generate random changes to objects hooked to top-level root by a/b/c/... key
Expand Down Expand Up @@ -196,13 +209,22 @@ def gen_testdb(outfs_path):
# ----------------------------------------

from zodbtools.zodbdump import zodbdump
from zodbtools.test.testutil import zext_supported

def main():
# check that ZODB supports txn.extension_bytes; refuse to work if not.
if not zext_supported():
raise RuntimeError("gen_testdata must be used with ZODB that supports txn.extension_bytes")

out = "testdata/1"
gen_testdb("%s.fs" % out)
stor = FileStorage("%s.fs" % out, read_only=True)
with open("%s.zdump.ok" % out, "w") as f:
zodbdump(stor, None, None, out=f)
for zext in [True, False]:
dbname = out
if not zext:
dbname += "_!zext"
gen_testdb("%s.fs" % dbname, zext=zext)
stor = FileStorage("%s.fs" % dbname, read_only=True)
with open("%s.zdump.ok" % dbname, "w") as f:
zodbdump(stor, None, None, out=f)

if __name__ == '__main__':
main()
4 changes: 2 additions & 2 deletions zodbtools/test/test_commit.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@

# verify zodbcommit.
@func
def test_zodbcommit():
def test_zodbcommit(zext):
tmpd = mkdtemp('', 'zodbcommit.')
defer(lambda: rmtree(tmpd))

Expand All @@ -40,7 +40,7 @@ def test_zodbcommit():

# commit some transactions via zodbcommit and verify if storage dump gives
# what is expected.
t1 = Transaction(z64, ' ', b'user name', b'description ...', dumps({'a': 'b'}, _protocol), [
t1 = Transaction(z64, ' ', b'user name', b'description ...', zext(dumps({'a': 'b'}, _protocol)), [
ObjectData(p64(1), b'data1', 'sha1', sha1('data1')),
ObjectData(p64(2), b'data2', 'sha1', sha1('data2'))])

Expand Down
12 changes: 7 additions & 5 deletions zodbtools/test/test_dump.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,14 +27,16 @@

from os.path import dirname

from pytest import raises
from zodbtools.test.testutil import zext_supported
from pytest import raises, xfail

# verify zodbdump output against golden
def test_zodbdump():
tdir = dirname(__file__)
stor = FileStorage('%s/testdata/1.fs' % tdir, read_only=True)
def test_zodbdump(zext):
tdir = dirname(__file__)
zkind = '_!zext' if zext.disabled else ''
stor = FileStorage('%s/testdata/1%s.fs' % (tdir, zkind), read_only=True)

with open('%s/testdata/1.zdump.ok' % tdir) as f:
with open('%s/testdata/1%s.zdump.ok' % (tdir, zkind)) as f:
dumpok = f.read()

out = StringIO()
Expand Down
Binary file added zodbtools/test/testdata/1_!zext.fs
Binary file not shown.
Binary file added zodbtools/test/testdata/1_!zext.fs.index
Binary file not shown.
Binary file added zodbtools/test/testdata/1_!zext.zdump.ok
Binary file not shown.
63 changes: 63 additions & 0 deletions zodbtools/test/testutil.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (C) 2019 Nexedi SA and Contributors.
# Kirill Smelkov <[email protected]>
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
"""utilities for testing"""

from ZODB.FileStorage import FileStorage
from ZODB import DB
import transaction

from tempfile import mkdtemp
from shutil import rmtree
from golang import func, defer

# zext_supported checks whether ZODB supports txn.extension_bytes .
_zext_supported_memo = None
def zext_supported():
global _zext_supported_memo
if _zext_supported_memo is not None:
return _zext_supported_memo

_ = _zext_supported_memo = _zext_supported()
return _

@func
def _zext_supported():
tmpd = mkdtemp('', 'zext_check.')
defer(lambda: rmtree(tmpd))
dbfs = tmpd + '/1.fs'

stor = FileStorage(dbfs, create=True)
db = DB(stor)
conn = db.open()
root = conn.root()
root._p_changed = True

txn = transaction.get()
txn.setExtendedInfo('a', 'b')
txn.commit()

for last_txn in stor.iterator(start=stor.lastTransaction()):
break
else:
assert False, "cannot see details of last transaction"

assert last_txn.extension == {'a': 'b'}
return hasattr(last_txn, 'extension_bytes')

0 comments on commit 425e665

Please sign in to comment.