#!/usr/bin/env python
#############################################################################
##
## This file is part of Taurus
##
## http://taurus-scada.org
##
## Copyright 2011 CELLS / ALBA Synchrotron, Bellaterra, Spain
##
## Taurus is free software: you can redistribute it and/or modify
## it under the terms of the GNU Lesser General Public License as published by
## the Free Software Foundation, either version 3 of the License, or
## (at your option) any later version.
##
## Taurus is distributed in the hope that it will be useful,
## but WITHOUT ANY WARRANTY; without even the implied warranty of
## MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
## GNU Lesser General Public License for more details.
##
## You should have received a copy of the GNU Lesser General Public License
## along with Taurus. If not, see <http://www.gnu.org/licenses/>.
##
#############################################################################
'''Utility functions to deal with non-ideal (fuzzy) tests'''
[docs]def loopTest(testname, maxtries=100, maxfails=10):
'''Run a test `maxtries` times or until it fails `maxfails` times and
report the number of tries and failures.
:param testname: (str) test name. see:
:meth:`unittest.TestLoader.loadTestsFromName`
:param maxtries: (int) maximum number of runs
:param maxfails: (int) maximum number of failed runs
:return: (tuple) a tuple of ints: tries, failures
'''
from taurus.external import unittest
suite = unittest.defaultTestLoader.loadTestsFromName(testname)
runner = unittest.TextTestRunner(verbosity=0)
i, f = 0, 0
while f < maxfails and i < maxtries:
i += 1
result = runner.run(suite)
if not result.wasSuccessful():
f += 1
return i, f
[docs]def loopSubprocess(target, maxtries=100, maxfails=10, okvalues=(0,), args=(),
kwargs=None):
'''Run a callable as a subprocess `maxtries` times or until it fails
`maxfails` times and report the number of tries and failures.
The callable is run as a subprocess and it is considered to run fine if
the subprocess exit code is in the okValues list.
:param target: (callable) a callable test
:param maxtries: (int) maximum number of runs
:param maxfails: (int) maximum number of failed runs
:param okvalues: (seq) a sequence containing exit values of cmd which
are considered to be successful runs.
:param args: (seq) arguments for running the target function
:param kwargs: (dict) keyword arguments for running the target function
:return: (tuple) a tuple of ints: tries, failures
'''
if kwargs is None:
kwargs = {}
from multiprocessing import Process
i, f = 0, 0
while f < maxfails and i < maxtries:
i += 1
p = Process(target=target, args=args, kwargs=kwargs)
p.start()
p.join()
if p.exitcode not in okvalues:
f += 1
return i, f
[docs]def calculateTestFuzziness(test, maxtries=100, maxfails=10, **kwargs):
'''Estimate the fuzziness of a test by running it many times and counting
the failures. In this context, we assume that there is an underlying
problem and but that the test is not perfect and only fails (triggers the
problem) with a certain failure rate.
:param testname: (str) test name. see:
:meth:`unittest.TestLoader.loadTestsFromName`
:param maxtries: (int) maximum number of runs
:param maxfails: (int) maximum number of failed runs
:return: (tuple) a tuple (f,df,n) where f is the failure rate, df is its
standard deviation, and n is the number of consecutive
times that the test should be passed to have a confidence>99%%
that the bug is fixed'
'''
print ("Running the test %i times (or until it fails %i times)" +
"to estimate the failure rate") % (maxtries, maxfails)
import numpy
if isinstance(test, str):
tries, fails = loopTest(test, maxtries=maxtries, maxfails=maxfails)
else:
tries, fails = loopSubprocess(test, maxtries=maxtries,
maxfails=maxfails, **kwargs)
r = float(fails)/tries
dr = numpy.sqrt(fails)/tries
print 'Failure rate = %g +/- %g (%i/%i)'%(r, dr, fails, tries)
#calculating n using p-value=1% and failure rate with -1 sigma
n = numpy.ceil(numpy.log(.01)/numpy.log(1-(r-dr)))
print ('Number of consecutive times that the test should be passed ' +
'to have a confidence>99%% that the bug is fixed: %g') % n
return r, dr, n
if __name__ == "__main__":
def kk():
#exits with fail 1/3 of the times
from numpy.random import randint, seed
seed()
k = randint(3)
if not k:
exit(1)
return
print calculateTestFuzziness(kk)
# print calculateTestFuzziness('test_pytango_bug659.TestPyTango_Bug659')
#
# _, f = loopTest('test_pytango_bug659.TestPyTango_Bug659',
# maxtries=100, maxfails=1)