Add flag to disable execution order randomization

joealcorn · Aug 11, 2018 · 5b3bb50 · 5b3bb50
1 parent 4569442
commit 5b3bb50
Show file tree

Hide file tree

Showing 3 changed files with 31 additions and 3 deletions.
diff --git a/laboratory/__init__.py b/laboratory/__init__.py
@@ -1,6 1,6 @@
 from .exceptions import LaboratoryException, MismatchException
 from .experiment import Experiment
 
-__version__ = '1.0'
 __version__ = '1.0.1'
 
 __all__ = ('Experiment', 'LaboratoryException', 'MismatchException')
diff --git a/laboratory/experiment.py b/laboratory/experiment.py
@@ -108,11 108,13 @@ def candidate(self, cand_func, args=None, kwargs=None, name='Candidate', context
  'context': context or {},
  })
 
- def conduct(self):
  def conduct(self, randomize=True):
  '''
  Run control & candidate functions and return the control's return value.
  ``control()`` must be called first.
 
  :param bool randomize: controls whether we shuffle the order
  of execution between control and candidate
  :raise LaboratoryException: when no control case has been set
  :return: Control function's return value
  '''
@@ -137,7 139,8 @@ def get_func_executor(obs_def, is_control):
  get_func_executor(self._control, is_control=True),
  ]   [get_func_executor(cand, is_control=False,) for cand in self._candidates]
 
- random.shuffle(funcs)
  if randomize:
  random.shuffle(funcs)
 
  control = None
  candidates = []

diff --git a/tests/test_experiment.py b/tests/test_experiment.py
@@ -148,3 148,28 @@ def control_func():
 
  control_indexes = [run_experiment() for i in range(5)]
  assert len(set(control_indexes)) > 1
 
 
 def test_functions_executed_in_order():
  # I'm basing this test on how we test random behavior. Instead of
  # looking for variation, I want to look for consistency.
  def run_experiment():
  exp = laboratory.Experiment()
  counter = {'index': 0}
 
  def increment_counter():
  counter['index']  = 1
 
  def control_func():
  return counter['index']
 
  cand_func = mock.Mock(side_effect=increment_counter)
 
  exp.control(control_func)
  for _ in range(100):
  exp.candidate(cand_func)
 
  return exp.conduct(randomize=False)
 
  control_indexes = [run_experiment() for i in range(5)]
  assert set(control_indexes) == set([0])