Big Refactoring (BR) of the test execution model draft
authorDavid Douard <david.douard@logilab.fr>
Fri, 14 Nov 2014 12:51:47 +0100
changeset 1740 f3137466457a
parent 1739 50514ad06479
child 1741 b89dc2e234ff
Big Refactoring (BR) of the test execution model Get rid of narval preprocessors in favor of Recipes - Each preprocessing recipe must be run from the "main" recipe (thus we add a ATest.exec_recipe() method that can be called from Recipe execution). - We add a (Repository, checkout_recipe, Recipe) relation, which is the Recipe that tells how to retrieve the code for a given Repository - Also add a (ProjectEnvironment, setup_recipe, Recip) dedicated to perform the installation for a project (compile and install) - Add implementations for these recipes for Python project - Rewrite the quick recipe using this new model
_apycotlib/atest.py
_apycotlib/repositories.py
entities.py
hooks.py
migration/postcreate.py
recipes.py
recipes/__init__.py
recipes/apycot.checkout.mercurial.py
recipes/apycot.quick.py
recipes/apycot.setup.distutils.py
schema.py
test/data/project1.zip
test/data/project2.zip
test/test_functional.py
test/unittest_apycot.py
test/unittest_entities.py
test/unittest_hooks.py
test/unittest_preprocessors.py
test/unittest_repositories.py
test/unittest_task.py
testutils.py
--- a/_apycotlib/atest.py	Thu Dec 11 22:18:16 2014 +0100
+++ b/_apycotlib/atest.py	Fri Nov 14 12:51:47 2014 +0100
@@ -14,7 +14,7 @@
 import os
 import os.path
 import sys
-import tempfile
+from tempfile import mkdtemp, NamedTemporaryFile
 from shutil import rmtree
 
 from logilab.common.proc import ResourceError
@@ -25,17 +25,6 @@
                        SUCCESS, SKIPPED, ERROR, KILLED)
 
 
-def substitute(value, substitutions):
-    if hasattr(value, 'replace') and value:
-        for key, val in substitutions.iteritems():
-            value = value.replace('${%s}' % key, val)
-    return value
-
-def substitute_dict(dict, substitutions):
-    for key, value in dict.iteritems():
-        dict[key] = substitute(value, substitutions)
-    return dict
-
 def clean_path(path):
     """remove trailing path separator from path"""
     if path and path[-1] == os.sep:
@@ -65,24 +54,23 @@
     def __init__(self, texec, writer):
         options = options_dict(texec['options'])
         # directory where the test environment will be built
-        self.tmpdir = tempfile.mkdtemp(dir=options.get('test_dir'))
+        self.tmpdir = mkdtemp(dir=options.get('test_dir'))
         # notify some subprocesses they're executed by apycot through an
         # environment variable
         os.environ['APYCOT_ROOT'] = self.tmpdir
         # test config / tested project environment
         self.texec = texec
         self.tconfig = texec['configuration']
-        self.environment = texec['environment']
+        self.penvironment = texec['environment']
         # IWriter object
         self.writer = writer
         # local caches
-        self._configs = {}
+        self._configs_cache = {}
         self._repositories = {}
         # environment variables as a dictionary
         self.environ = self.tconfig['apycot_process_environment']
-        self.environ.update(self.environment['apycot_process_environment'])
+        self.environ.update(self.penvironment['apycot_process_environment'])
         self.environ.setdefault('LC_ALL', 'fr_FR.UTF-8') # XXX force utf-8
-        self._substitute(self.environment, self.environ)
         # Track environment change to be able to restore it later.
         # Notice sys.path is synchronized with the PYTHONPATH environment variable
         self._tracks = {}
@@ -100,53 +88,47 @@
     def __str__(self):
         return repr(self.apycot_repository())
 
-    def _substitute(self, pe, configdict):
-        substitute_dict(configdict,
-                        {'NAME': pe['name'], 'TESTDIR': self.tmpdir,
-                         'SRCDIR': self.apycot_repository(pe).co_path})
-
     # resource accessors #######################################################
 
-    def apycot_config(self, pe=None):
-        if pe is None:
-            pe = self.environment
+    def get_entity(self, eid):
+        """Retrieve the entity from cw"""
+        return self.writer.cnxh.rql('Any E WHERE E eid %s' % eid,
+                                    vid='ejsonexport')
+
+    def rql(self, rql, **args):
+        return self.writer.cnxh.rql(rql, **args)
+
+    def apycot_config(self, pe_eid=None):
+        if pe_eid is None:
+            pe_eid = self.penvironment['eid']
         try:
-            return self._configs[pe['eid']]
+            return self._configs_cache[pe_eid]
         except KeyError:
             config = self.writer.cnxh.rql('Any E WHERE E eid %s' % self.tconfig['eid'],
                                           vid='apycot.get_configuration',
-                                          environment=pe['eid']).json()[0]
-            self._configs[pe['eid']] = config
-            self._substitute(pe, config)
+                                          environment=pe_eid).json()[0]
+            self._configs_cache[pe_eid] = config
             return config
 
-    def apycot_repository(self, pe=None):
-        if pe is None:
-            pe = self.environment
+    def exec_recipe(self, recipe, working_directory=None, **kwargs):
+        cwd = os.getcwd()
+        resp = self.writer.cnxh.rql('Recipe R WHERE R name "%s"' % recipe,
+                                    vid='ejsonexport')
+        if resp.status_code > 399:
+            msg = "Cannot retrieve recipe %s: %s" % (recipe, resp.text)
+            self.writer.error(msg)
+            raise ValueError(msg)
+        recipe = resp.json()[0]
+
         try:
-            return self._repositories[pe['eid']]
-        except KeyError:
-            from apycotlib.repositories import get_repository
-            if 'repository' not in pe or not pe['repository'] :
-                raise Exception('Project environment %s has no repository' % pe['title'])
-            repdef = {'repository': pe['repository'],
-                      'path': pe['vcs_path'],
-                      'branch': self.texec['branch']}
-            # don't overwrite branch hardcoded on the environment: have to be
-            # done here, not only in when starting plan (eg in entities.py)
-            # since project env may not be the tested project env
-            pecfg = pe['apycot_configuration']
-            if pecfg.get('branch'):
-                repdef['branch'] = pecfg['branch']
-            apyrep = get_repository(repdef)
-            self._repositories[pe['eid']] = apyrep
-            return apyrep
-
-    def project_path(self, subpath=False):
-        path = self.apycot_repository().co_path
-        if subpath and self.apycot_config().get('subpath'):
-            return os.path.join(path, self.apycot_config()['subpath'])
-        return path
+            if working_directory is not None:
+                os.chdir(working_directory)
+            with NamedTemporaryFile(suffix='.py') as f:
+                f.write(recipe['script'])
+                f.flush()
+                execfile(f.name, kwargs)
+        finally:
+            os.chdir(cwd)
 
     # test initialisation / cleanup ############################################
 
@@ -226,51 +208,10 @@
 
     # api to call a particular preprocessor / checker #########################
 
-    def checkout(self, pe):
-        vcsrepo = self.apycot_repository(pe)
-        cocmd = vcsrepo.co_command()
-        if cocmd:
-            Command(self.writer, cocmd, raises=True, shell=True,
-                    cwd=self.tmpdir).run()
-        movebranchcmd = vcsrepo.co_move_to_branch_command()
-        if movebranchcmd:
-            Command(self.writer, movebranchcmd, shell=True,
-                    cwd=self.tmpdir).run()
-        self.writer.link_to_revision(pe, vcsrepo)
-        self.writer.refresh_log()
-
-    def call_preprocessor(self, pptype, penv):
-        cfg = self.apycot_config(penv)
-        ppid = cfg.get(pptype)
-        if ppid is not None:
-            # fetch preprocessors options set on the project environment
-            preprocessor = get_registered('preprocessor', ppid)(self.writer, cfg)
-        else:
-            # XXX log?
-            return
-        path = self.apycot_repository(penv).co_path
-        dependency = path != self.project_path()
-        msg = 'running preprocessor %(pp)s to perform %(pptype)s'
-        msg_data = {'pptype': pptype,
-                    'pp': preprocessor.id,}
-        if dependency:
-            msg + ' on dependency %(pe)s'
-            msg_data['pe'] = penv['name']
-        self.writer.info(msg % msg_data, path=path)
-        try:
-            preprocessor.run(self, path)
-        except Exception, ex:
-            msg = '%s while running preprocessor %s: %s'
-            self.writer.fatal(msg, ex.__class__.__name__, preprocessor.id, ex,
-                              path=path, tb=True)
-            self._failed_pp.add(pptype)
-            self.global_status = min(self.global_status, ERROR)
-
     def run_checker(self, id, displayname=None, nonexecuted=False, **kwargs):
         """run all checks in the test environment"""
         options = self.options.copy()
         options.update(kwargs)
-        self._substitute(self.environment, options)
         check_writer = self.writer.make_check_writer()
         if nonexecuted:
             check_writer.start(id)
--- a/_apycotlib/repositories.py	Thu Dec 11 22:18:16 2014 +0100
+++ b/_apycotlib/repositories.py	Fri Nov 14 12:51:47 2014 +0100
@@ -15,60 +15,51 @@
 from apycotlib import register, get_registered, ConfigError
 
 
-SUPPORTED_REPO_TYPES = ('mercurial', 'subversion')
-
-
-def get_repository(attrs):
+def get_repository(**attrs):
     """factory method: return a repository implementation according to
     <attrs> (a dictionary)
     """
     repo_type = attrs['repository']['type']
-    assert repo_type in SUPPORTED_REPO_TYPES, repo_type
     return get_registered('repository', repo_type)(attrs)
 
-class VersionedRepository:
-    """base class for versionned repository"""
+class VCSRepository(objet):
+    """base class for clonable repository"""
 
     id = None
     default_branch = None
 
-    def __init__(self, attrs):
-        try:
-            self.repository = attrs.pop('repository')
-        except KeyError, ex:
-            raise ConfigError('Missing %s option: %s' % (ex, attrs))
+    def __init__(self, repository, changeset):
+        self.repository = repository
         if not self.repository:
-            raise ConfigError('Repository must be specified (%s)' % (attrs,))
-        self.path = attrs.pop('path', '')
-        branch = attrs.pop('branch', None)
-        if branch is None:
-            branch = self.default_branch
-        self.branch = branch
-        self.ref_repo = self._ref_repo()
+            raise ConfigError('Repository must be specified')
+        self.cset = changeset
         if not self.ref_repo:
             raise ConfigError('Missing information to checkout repository %s'
                               % self.repository)
-        # absolute path where the project will be located in the test
-        # environment
-        self.co_path = osp.join(environ['APYCOT_ROOT'], self._co_path())
 
     def __eq__(self, other):
         return (isinstance(other, self.__class__) and
                 self.repository == other.repository and
-                self.path == other.path and
-                self.branch == other.branch)
+                self.path == other.path)
 
     def __ne__(self, other):
         return not self == other
 
     def __repr__(self):
         """get a string synthetizing the location"""
-        myrepr = '%s:%s' % (self.id, self.repository['source_url'] or self.repository['path'])
+        myrepr = '%s:%s' % (self.id, self.ref_repo)
+        myrepr += '@%s' % self.cset
+        return myrepr
+
+    @property
+    def co_path(self):
+        """return the path where the project will be located in the test
+        environment
+        """
+        copath = split_url_or_path(self.ref_repo)[1]
         if self.path:
-            myrepr += '/' + self.path
-        if self.branch:
-            myrepr += '@%s' % self.branch
-        return myrepr
+            copath = osp.join(copath, self.path)
+        return osp.join(environ['APYCOT_ROOT'], copath)
 
     def co_command(self, quiet=True):
         """return a command that may be given to os.system to check out a given
@@ -76,46 +67,15 @@
         """
         raise NotImplementedError()
 
-    def co_move_to_branch_command(self, quiet=True):
-        return None
 
-    def normalize_date(self, from_date, to_date):
-        """get dates as float or local time and return the normalized dates as
-        local time tuple to fetch log information from <from_date> to <to_date>
-        included
-        """
-        if isinstance(from_date, float):
-            from_date = localtime(from_date)
-        if isinstance(to_date, float):
-            to_date = localtime(to_date)
-        return (from_date, to_date)
-
-    def changeset(self):
-        """return changeset of the working directory"""
-        return None
-
-    def revision(self):
-        """return revision number of the working directory"""
-        return None
-
-
-class HGRepository(VersionedRepository):
+class HGRepository(VCSRepository):
     """extract sources/information for a project from a Mercurial repository"""
     id = 'mercurial'
-    default_branch = "default"
 
-    def _ref_repo(self):
+    @property
+    def ref_repo(self):
         return self.repository['source_url']
 
-    def _co_path(self):
-        """return the path where the project will be located in the test
-        environment
-        """
-        copath = split_url_or_path(self.ref_repo)[1]
-        if self.path:
-            copath = osp.join(copath, self.path)
-        return copath
-
     def co_command(self, quiet=True):
         """return a command that may be given to os.system to check out a given
         package
@@ -124,26 +84,6 @@
             return "hg clone -q %s && hg -R %s up '::. and public()'" % (self.ref_repo, self.co_path)
         return "hg clone %s && hg -R %s up '::. and public()'" % (self.ref_repo, self.co_path)
 
-    def co_move_to_branch_command(self, quiet=True):
-        # if branch doesn't exist, stay in default
-        if self.branch:
-            return "hg -R %s up 'first(id(%s) + max(branch(%s) and public()))'" % (
-                    self.co_path, self.branch, self.branch)
-        return None
-
-    def changeset(self):
-        import hglib
-        with hglib.open(self.co_path) as repo:
-            parents = repo.parents()
-        #assert len(parents) == 0 ?
-        return parents[0].node[:12]
-
-    def revision(self):
-        import hglib
-        with hglib.open(self.co_path) as repo:
-            parents = repo.parents()
-        #assert len(parents) == 0 ?
-        return parents[0].rev
 
 register('repository', HGRepository)
 
--- a/entities.py	Thu Dec 11 22:18:16 2014 +0100
+++ b/entities.py	Fri Nov 14 12:51:47 2014 +0100
@@ -102,6 +102,9 @@
         return text_to_dict(self.check_config)
 
     def apycot_configuration(self):
+        """Agregate the configuration options of all parents
+        (according the refinement_of relation) in one dictionnary
+        """
         return self._regroup_dict('my_apycot_configuration')
 
     @property
@@ -109,6 +112,9 @@
         return text_to_dict(self.check_environment)
 
     def apycot_process_environment(self):
+        """Agregate the environment variables of all parents
+        (according the refinement_of relation) in one dictionnary
+        """
         return self._regroup_dict('my_apycot_process_environment')
 
 
--- a/hooks.py	Thu Dec 11 22:18:16 2014 +0100
+++ b/hooks.py	Fri Nov 14 12:51:47 2014 +0100
@@ -95,6 +95,19 @@
                 ComputeStartModeOp(self.cnx, tc=refined)
 
 
+# automatic recipe attachements
+
+class CheckoutRecipeHook(hook.Hook):
+    __regid__ = 'apycot.chekout_recipe'
+    __select__ = hook.Hook.__select__ & is_instance('Repository')
+    events = ('after_add_entity',)
+    def __call__(self):
+        rset = self._cw.execute('Recipe R WHERE R name %(name)s',
+                                {'name': u'apycot.checkout.%s' % self.entity.type})
+        if rset:
+            self.entity.cw_set(checkout_recipe=rset.one())
+
+
 # automatic test launching #####################################################
 
 class ServerStartupHook(hook.Hook):
--- a/migration/postcreate.py	Thu Dec 11 22:18:16 2014 +0100
+++ b/migration/postcreate.py	Fri Nov 14 12:51:47 2014 +0100
@@ -9,10 +9,12 @@
                   requiredgroups=('managers',))
 
 # workflows don't consider schema inheritance, so we need to set it explicitly
-rql('SET WF workflow_of TE, TE default_workflow WF WHERE WF workflow_of P, P name "Plan", TE name "TestExecution"')
+rql('SET WF workflow_of TE, TE default_workflow WF WHERE WF workflow_of P, '
+    'P name "Plan", TE name "TestExecution"')
 commit()
 
+
+print " RECIPES ".center(60, "=")
 from cubes.apycot import recipes
-recipes.create_quick_recipe(session)
-recipes.create_full_recipe(session)
+recipes.create_recipes(session)
 commit()
--- a/recipes.py	Thu Dec 11 22:18:16 2014 +0100
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,82 +0,0 @@
-"""functions to create defaut apycot recipes"""
-
-quick_script = u'''
-from checkers.apycot import python # trigger registration
-from apycotlib import narvalactions as na
-# `plan` (narvalbot.engine.Plan) exists in the globals
-
-with na.apycot_environment(plan) as test:
-    na.install_environment(test)
-    checker, status = test.run_checker('pyunit')
-'''
-def create_quick_recipe(session):
-    return session.create_entity('Recipe', name=u'apycot.recipe.quick',
-                                 script=quick_script)
-
-full_script = u'''
-from checkers.apycot import python # trigger registration
-from apycotlib import registered, narvalactions as na
-from apycotlib import ERROR
-# `plan` (narvalbot.engine.Plan) exists in the globals
-
-with na.apycot_environment(plan) as test:
-    na.install_environment(test)
-    if registered('checker', 'pylint'): # pylint may not be available
-        checker, status = test.run_checker('pylint')
-    checker, status = test.run_checker('pyunit', pycoverage=True)
-    if status > ERROR:
-        checker, status = test.run_checker('pycoverage',
-                                            coverage_data=checker.coverage_data)
-
-'''
-def create_full_recipe(session):
-    return session.create_entity('Recipe', name=u'apycot.recipe.full',
-                                 script=full_script)
-
-scenario_runner_script = u'''
-from checkers.apycot.scenarios import ScriptRunner
-from apycotlib import narvalactions as na
-# `plan` (narvalbot.engine.Plan) exists in the globals
-
-class ScenarioChecker(ScriptRunner):
-    id = "scenario_checker"
-    def filename_filter(self, dirpath, dirnames, filenames):
-        """
-        this function takes parameters from os.walk and has two objectives:
-          - from the dirnames, prune folders you do not want to explore
-            using dirnames.remove(x)
-          - remove all the filenames which should not be run from filenames in
-            the same way.
-        """
-        for dirname in dirnames[:]:
-            if dirname in ('.hg', '.git', '.svn'):
-                dirnames.remove(dirname)
-        for filename in filenames[:]:
-            if not (filename.endswith('.py') and filename.startswith('scenario_')):
-                filenames.remove(filename)
-
-register('checker', ScenarioChecker)
-
-with na.apycot_environment(plan) as test:
-    na.install_environment(test)
-    checker, status = test.run_checker('scenario_checker')
-'''
-
-def create_scenrario_filter_recipe(session):
-    return session.create_entity('Recipe', name=u'apycot.recipe.scenario_runner',
-                                 script=scenario_runner_script)
-
-
-pypi_script = u'''
-# A simple recipe that uploads a project on pypi (if unit tests are OK)
-
-from checkers.apycot import pypi # must be first (trigger registration)
-from apycotlib import narvalactions as na
-# `plan` (narvalbot.engine.Plan) exists in the globals
-
-with na.apycot_environment(plan) as test:
-    test.checkout(plan)
-    checker, status = test.run_checker('pyunit')
-    if status > ERROR:
-        checker, status = test.run_checker('pypi.upload')
-'''
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/recipes/__init__.py	Fri Nov 14 12:51:47 2014 +0100
@@ -0,0 +1,19 @@
+#
+from os import listdir
+from os.path import join, dirname
+
+def create_recipes(cnx):
+    recipes = []
+    rdir = dirname(__file__)
+    for recipe in listdir(rdir):
+        if recipe.endswith('.py') and not recipe.startswith('_'):
+            rname = recipe[:-3]
+            try:
+                script = open(join(rdir, recipe)).read()
+                r = cnx.create_entity('Recipe',
+                                      name=rname.decode('utf-8'),
+                                      script=script.decode('utf-8'))
+                recipes.append(r)
+            except Exception as exc:
+                cnx.warning('Failed to create Recipe %s (%s)'%(rname, exc))
+    return recipes
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/recipes/apycot.checkout.mercurial.py	Fri Nov 14 12:51:47 2014 +0100
@@ -0,0 +1,17 @@
+# normally executed with following builtin variables:
+# :source_url:
+# :rev:
+# :dstdir:
+
+import os
+import hglib
+
+os.environ['HGRCPATH'] = os.devnull
+
+configs = [('ui.username', 'narval'),
+           ('phases.publish', 'False'),
+           ]
+
+if not os.path.exists(dst_dir):
+    os.makedirs(dst_dir)
+hglib.clone(source_url, dst_dir, updaterev=rev)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/recipes/apycot.quick.py	Fri Nov 14 12:51:47 2014 +0100
@@ -0,0 +1,24 @@
+# an apycot recipe that run unit tests for python projects
+import os
+from os.path import join
+from checkers.apycot import python # trigger registration
+from apycotlib import narvalactions as na
+# `plan` (narvalbot.engine.Plan) exists in the globals
+
+with na.apycot_environment(plan) as test:
+    for pe, rev in test.dependencies():
+        # first need to retrieve the sources of the dependency
+        vcsrepo = pe['repository']
+        # use the correct recipe to make the checkout and ensure the working
+        # directory is at the selected revision
+        src = join(test.tmpdir, 'src', str(vcsrepo['eid']))
+        test.exec_recipe(vcsrepo['checkout_recipe'], rev=rev,
+                         dst_dir=src,
+                         source_url=vcsrepo['source_url'])
+        # now perform the setup process (build and install)
+        test.exec_recipe(pe['setup_recipe'],
+                         prefix=test.tmpdir,
+                         writer=test.writer,
+                         wdir=src)
+
+    checker, status = test.run_checker('pyunit')
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/recipes/apycot.setup.distutils.py	Fri Nov 14 12:51:47 2014 +0100
@@ -0,0 +1,23 @@
+# normally executed with following builtin variables:
+# :prefix: where to install
+# :writer: log reporter
+# :wdir: working directory
+
+from apycotlib import Command
+
+# ensure environment variables are OK
+test.update_env(wdir, 'PATH',
+                join(wdir, 'sbin'),
+                os.pathset)
+test.update_env(wdir, 'PATH',
+                join(wdir, 'bin'),
+                os.pathset)
+test.update_env(wdir, 'PYTHONPATH',
+                join(wdir, 'lib', 'python'),
+                os.pathset)
+test.update_env(wdir, 'LD_LIBRARY_PATH',
+                join(wdir, 'lib'),
+                os.pathset)
+
+cmdargs = ['python', 'setup.py', 'install', '--home', prefix]
+Command(writer, cmdargs, raises=True, cwd=wdir)
--- a/schema.py	Thu Dec 11 22:18:16 2014 +0100
+++ b/schema.py	Fri Nov 14 12:51:47 2014 +0100
@@ -218,17 +218,28 @@
     description = _('vcsfile repository holding the source code')
 
 
-# class needs_checkout(RelationDefinition):
-#     __permissions__ = {
-#         'read':   ('managers', 'users', 'guests', 'narval'),
-#         'add':    CONF_WRITE_GROUPS,
-#         'delete': CONF_WRITE_GROUPS,
-#         }
-#     subject = 'ProjectEnvironment'
-#     object = 'ProjectEnvironment'
-#     description = _('project\'s environments that should be installed from '
-#                     'their repository to execute test for the environment or with this configuration')
-#     #constraints=[RQLConstraint('NOT S identity O')]
+class checkout_recipe(RelationDefinition):
+    __permissions__ = {
+        'read':   ('managers', 'users', 'guests', 'narval'),
+        'add':    CONF_WRITE_GROUPS,
+        'delete': CONF_WRITE_GROUPS,
+        }
+    subject = 'Repository'
+    object = 'Recipe'
+    cardinality = '?*'
+    description = _('Recipe used to retrieve the source code from this repository')
+
+class setup_recipe(RelationDefinition):
+    __permissions__ = {
+        'read':   ('managers', 'users', 'guests', 'narval'),
+        'add':    CONF_WRITE_GROUPS,
+        'delete': CONF_WRITE_GROUPS,
+        }
+    subject = 'ProjectEnvironment'
+    object = 'Recipe'
+    cardinality = '?*'
+    description = _('Recipe used to perform the building and the installation of the project')
+
 
 class pe_refinement_of(RelationDefinition):
     __permissions__ = {
@@ -240,7 +251,7 @@
     cardinality = '?*'
     subject = 'ProjectEnvironment'
     object = 'ProjectEnvironment'
-    #constraints=[RQLConstraint('NOT S identity O')]
+
 
 class tc_refinement_of(pe_refinement_of):
     subject = 'TestConfig'
Binary file test/data/project1.zip has changed
Binary file test/data/project2.zip has changed
--- a/test/test_functional.py	Thu Dec 11 22:18:16 2014 +0100
+++ b/test/test_functional.py	Fri Nov 14 12:51:47 2014 +0100
@@ -7,25 +7,18 @@
 
 import cubicweb.devtools
 import cubes.apycot.testutils as utils
-from cubes.apycot.recipes import full_script
-from cubes.vcsfile.testutils import HGRCMixin
 
 DATA = osp.join(osp.dirname(__file__), 'data')
 os.environ['HGRCPATH'] = os.devnull
 
-class ApycotTC(HGRCMixin, utils.ApycotBaseTC):
-    _repo_path = u'project'
-
-    def setUp(self):
-        utils.setup_repos(DATA, self._repo_path)
-        super(ApycotTC, self).setUp()
-
+class ApycotTC(utils.ApycotBaseTC):
+    _repo_path = (u'project',)
 
     def test_quick_recipe(self):
         with self.admin_access.client_cnx() as cnx:
-            lgc = cnx.entity_from_eid(self.lgc)
-            lgce = cnx.entity_from_eid(self.lgce)
-            te = lgc.start(lgce).eid
+            tc = cnx.find('TestConfig', name='tc_project1').one()
+            pe = cnx.fond('ProjectEnviironment', name='pe_project1').one()
+            te = tc.start(pe).eid
             cnx.commit()
         self.run_plan(te)
         with self.admin_access.client_cnx() as cnx:
--- a/test/unittest_apycot.py	Thu Dec 11 22:18:16 2014 +0100
+++ b/test/unittest_apycot.py	Fri Nov 14 12:51:47 2014 +0100
@@ -25,14 +25,16 @@
         self.options_def = dict( (k, {}) for k in options )
 
 class ApycotTC(ApycotBaseTC):
+    _repo_path = (u'project1',)
 
     def setUp(self):
         super(ApycotBaseTC, self).setUp()
         with self.admin_access.client_cnx() as cnx:
-            lgc = cnx.entity_from_eid(self.lgc)
-            lgce = cnx.entity_from_eid(self.lgce)
-            te = lgc.start(lgce).eid
+            pe = cnx.find('ProjectEnvironment').one()
+            tc = cnx.find('TestConfig', name='tc_project1').one()
+            te = tc.start(pe).eid
             cnx.commit()
+
         cnxh = HTTPConnectionHandler('narval')
         writer = TestDataWriter(cnxh, te)
         writer.start()
@@ -53,7 +55,8 @@
 
     def test_writer_log_content(self):
         with self.admin_access.client_cnx() as cnx:
-            checks = cnx.execute('Any X, N ORDERBY N WHERE X is CheckResult, X name N')
+            checks = cnx.execute('Any X, N ORDERBY N WHERE X is CheckResult, '
+                                 'X name N')
             self.assertEqual(len(checks), 2)
             self.assertEqual(checks.get_entity(0, 0).log_file[0].data.getvalue(),
                              '20\t\t\toption=value<br/>40\t\t\tbouh<br/>50\t\t\tdi&amp;d<br/>')
@@ -63,7 +66,8 @@
     def test_log_formatting_first_check(self):
         stream = []
         with self.admin_access.web_request() as req:
-            checks = req.execute('Any X, N ORDERBY N WHERE X is CheckResult, X name N')
+            checks = req.execute('Any X, N ORDERBY N WHERE X is CheckResult, '
+                                 'X name N')
             log_to_html(req, '',
                         checks.get_entity(0, 0).log_file[0].data.read(),
                         stream.append)
--- a/test/unittest_entities.py	Thu Dec 11 22:18:16 2014 +0100
+++ b/test/unittest_entities.py	Fri Nov 14 12:51:47 2014 +0100
@@ -21,38 +21,39 @@
 
 
 class ApycotConfigTC(ApycotBaseTC):
+    _repo_path = (u'project1',)
 
     def setup_database(self):
         ApycotBaseTC.setup_database(self)
         with self.admin_access.repo_cnx() as cnx:
-            self.lgd = self.add_test_config(cnx, u'lgd', check_config=None,
-                                            env=self.lgce, group=self.pyp).eid
+            pyp = cnx.find('TestConfig', name='PYTHONPACKAGE').one()
+            pe = cnx.find('ProjectEnvironment').one()
+            self.add_test_config(cnx, u'tcd', check_config=None,
+                                 env=pe, group=pyp)
             cnx.commit()
 
     def test_refinement_base(self):
         with self.admin_access.client_cnx() as cnx:
-            lgce = cnx.entity_from_eid(self.lgce)
-            lgc = cnx.entity_from_eid(self.lgc)
-            lgd = cnx.entity_from_eid(self.lgd)
-            self.assertEqual(lgce.apycot_process_environment(),
+            pe = cnx.find('ProjectEnvironment').one()
+            pyp = cnx.find('TestConfig', name='PYTHONPACKAGE').one()
+            tcd = cnx.find('TestConfig', name='tcd').one()
+            self.assertEqual(pe.apycot_process_environment(),
                              {'DISPLAY': ':2.0', 'SETUPTOOLS': '1'})
-            self.assertEqual(lgc.apycot_process_environment(),
+            self.assertEqual(pyp.apycot_process_environment(),
                              {'DISPLAY': ':1.0', 'NO_SETUPTOOLS': '1'})
 
-            self.assertEqual(lgd.apycot_configuration(lgce),
-                              {'install': 'python_setup',
-                               'python_lint_treshold': '7',
+            self.assertEqual(tcd.apycot_configuration(pe),
+                              {'python_lint_treshold': '7',
                                'python_lint_ignore': 'thirdparty',
                                'python_test_coverage_treshold': '70',
                                'env-option': 'value'})
 
     def test_refinement_override(self):
         with self.admin_access.client_cnx() as cnx:
-            lgce = cnx.entity_from_eid(self.lgce)
-            lgc = cnx.entity_from_eid(self.lgc)
-            self.assertEqual(lgc.apycot_configuration(lgce),
-                              {'install': 'python_setup',
-                               'python_lint_treshold': '8',
+            pe = cnx.find('ProjectEnvironment').one()
+            tc = cnx.find('TestConfig', name='tc_project1').one()
+            self.assertEqual(tc.apycot_configuration(pe),
+                              {'python_lint_treshold': '8',
                                'python_lint_ignore': 'thirdparty',
                                'python_test_coverage_treshold': '70',
                                'pouet': '5',
@@ -60,16 +61,17 @@
 
     def test_all_check_results(self):
         with self.admin_access.client_cnx() as cnx:
-            lgce = cnx.entity_from_eid(self.lgce)
-            lgc = cnx.entity_from_eid(self.lgc)
-            ex1eid = lgc.start(lgce, check_duplicate=False).eid
-            ex2eid = lgc.start(lgce, check_duplicate=False).eid
+            pe = cnx.find('ProjectEnvironment').one()
+            tc = cnx.find('TestConfig', name='tc_project1').one()
+            ex1eid = tc.start(pe, check_duplicate=False).eid
+            ex2eid = tc.start(pe, check_duplicate=False).eid
             cnx.commit()
 
         with self.new_access('narval').client_cnx() as cnx:
             ex1 = cnx.entity_from_eid(ex1eid)
             ex2 = cnx.entity_from_eid(ex2eid)
-            self.dumb_execution(cnx, ex1, [('unittest', 'success'), ('coverage', 'success')])
+            self.dumb_execution(cnx, ex1, [('unittest', 'success'),
+                                           ('coverage', 'success')])
             cnx.commit()
             covcr = ex1.check_result_by_name('coverage').eid
             self.dumb_execution(cnx, ex2, [('unittest', 'failure')])
@@ -78,24 +80,25 @@
             cnx.commit()
 
         with self.admin_access.client_cnx() as cnx:
-            lgc = cnx.entity_from_eid(self.lgc)
-            self.assertEqual([cr.eid for cr in all_check_results(lgc)],
+            tc = cnx.find('TestConfig', name='tc_project1').one()
+            self.assertEqual([cr.eid for cr in all_check_results(tc)],
                              [covcr, ucr])
 
     def test_duplicated_tc_same_env(self):
         tcncstrs = self.schema['TestConfig'].rdef('name').constraints
         self.assertEqual([cstr.type() for cstr in tcncstrs], ['RQLUniqueConstraint', 'SizeConstraint'])
         with self.admin_access.client_cnx() as cnx:
-            cnx.create_entity('TestConfig', name=u'lgd', use_environment=self.lgce)
+            pe = cnx.find('ProjectEnvironment').one()
+            cnx.create_entity('TestConfig', name=u'lgd', use_environment=pe)
             self.assertRaises(ValidationError, cnx.commit)
 
     def test_status_change(self):
         with self.admin_access.client_cnx() as cnx:
-            lgce = cnx.entity_from_eid(self.lgce)
-            lgc = cnx.entity_from_eid(self.lgc)
-            ex1eid = lgc.start(lgce, check_duplicate=False).eid
-            ex2eid = lgc.start(lgce, check_duplicate=False).eid
-            ex3eid = lgc.start(lgce, check_duplicate=False).eid
+            pe = cnx.find('ProjectEnvironment').one()
+            tc = cnx.find('TestConfig', name='tc_project1').one()
+            ex1eid = tc.start(pe, check_duplicate=False).eid
+            ex2eid = tc.start(pe, check_duplicate=False).eid
+            ex3eid = tc.start(pe, check_duplicate=False).eid
             cnx.commit()
 
         with self.new_access('narval').client_cnx() as cnx:
@@ -125,22 +128,6 @@
                                                ex3.check_result_by_name('unittest').eid),
                                   'coverage': (ex2.check_result_by_name('coverage').eid,
                                                ex3.check_result_by_name('coverage').eid)})
-    # XXX  to backport
-    # def test_branch_for_pe(self):
-    #     #check that branch defined in ProjectEnvironement are propertly retrieved
-
-    #     data = {
-    #         'cc': self.lgce.check_config + '\nbranch=toto',
-    #         'e': self.lgce.eid,
-
-    #     }
-    #     self.execute('SET PE check_config %(cc)s WHERE PE eid %(e)s', data)
-
-    #     entity = self.execute('Any PE WHERE PE eid %(e)s', data).get_entity(0,0)
-
-    #     repo_def = entity.apycot_repository_def
-    #     self.assertIn('branch', repo_def)
-    #     self.assertEqual(repo_def['branch'], 'toto')
 
 
 if __name__ == '__main__':
--- a/test/unittest_hooks.py	Thu Dec 11 22:18:16 2014 +0100
+++ b/test/unittest_hooks.py	Fri Nov 14 12:51:47 2014 +0100
@@ -9,7 +9,6 @@
 from cubicweb import Binary
 
 from cubes.vcsfile import bridge
-from cubes.vcsfile.testutils import HGRCMixin
 from cubes.apycot.hooks import start_period_tests
 from cubes.apycot.testutils import ApycotBaseTC
 
@@ -130,16 +129,17 @@
             self.assertNotIn('anon', [u.login for u in plan.nosy_list])
 
 
-class StartTestTC(HGRCMixin, ApycotBaseTC):
-
+class StartTestTC(ApycotBaseTC):
+    _repo_path = (u'project1',)
+    
     def setUp(self):
         super(StartTestTC, self).setUp()
         r = self.repo
         with self.admin_access.repo_cnx() as cnx:
-            vcsrepo = cnx.entity_from_eid(self.vcsrepo)
-            r.path = vcsrepo.localcachepath
-        if os.path.exists(r.path):
-            shutil.rmtree(r.path)
+            vcsrepo = cnx.find('Repository').one()
+            path = vcsrepo.localcachepath
+        return
+    
         os.system('hg init %s' % r.path)
         os.system('echo data > %s/tutu.png' % r.path)
         os.system('hg -R %s add %s/tutu.png' % (r.path, r.path))
@@ -148,11 +148,6 @@
         with r.internal_cnx() as cnx:
             bridge.import_content(cnx, commitevery=1, raise_on_error=True)
 
-    def tearDown(self):
-        r = self.repo
-        os.system('rm -rf %s/.hg' % r.path)
-        os.system('rm -rf %s/*.png' % r.path)
-        super(StartTestTC, self).tearDown()
 
     def get_tc(self, cnx, period=None):
         rql = ('Any PEN, TCN, TCS  WHERE TC in_state S, '
@@ -203,14 +198,16 @@
         """ check the on new revision start mode. Run all testconfigs, add new
         revision, re-run and check there are new test configs"""
         with self.admin_access.client_cnx() as cnx:
-            lgc = cnx.entity_from_eid(self.lgc)
-            lgc.cw_set(start_mode=u'on new revision')
-            lgc2 = self.add_test_config(cnx, u'lgc2',
-                                        start_mode=u'manual',
-                                        env=self.lgce,
-                                        use_recipe=self.recipe)
+            pe = cnx.find('ProjectEnvironment').one()
+            tc = cnx.find('TestConfig', name='tc_project1').one()
+            recipe = cnx.find('Recipe', name='apycot.quick')
+            tc.cw_set(start_mode=u'on new revision')
+            tc2 = self.add_test_config(cnx, u'tc2',
+                                       start_mode=u'manual',
+                                       env=pe,
+                                       use_recipe=recipe)
             ## same repo, branch stable
-            lgc3 = self.add_test_config(cnx, u'lgc3',
+            tc3 = self.add_test_config(cnx, u'tc3',
                                         start_mode=u'on new revision',
                                         check_config=u'branch=stable',
                                         env=self.lgce,
@@ -379,7 +376,7 @@
             self.assertNotEqual(u'waiting execution', te.status)
 
 
-class ComputedStartModeTC(HGRCMixin, ApycotBaseTC):
+class ComputedStartModeTC(ApycotBaseTC):
 
     def test_start_mode(self):
         with self.admin_access.client_cnx() as cnx:
@@ -399,5 +396,14 @@
             self.assertEqual(pyp.computed_start_mode, u'hourly')
             self.assertEqual(lgc.computed_start_mode, u'hourly')
 
+class ReposirotyRecipeTC(ApycotBaseTC):
+
+    def test_reposiroty_has_recipe(self):
+        with self.admin_access.client_cnx() as cnx:
+            repo = cnx.create_entity('Repository', type=u'mercurial',
+                                     source_url=u'file://path/to/repo',)
+            cnx.commit()
+            self.assertEqual('apycot.checkout.mercurial', repo.checkout_recipe[0].name)
+
 if __name__ == '__main__':
     unittest_main()
--- a/test/unittest_preprocessors.py	Thu Dec 11 22:18:16 2014 +0100
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,103 +0,0 @@
-# -*- coding: utf-8 -*-
-import os
-import json
-
-from logilab.common.testlib import unittest_main
-import cubicweb.devtools
-
-from cubes.apycot.testutils import ApycotBaseTC
-from apycotlib import atest, writer, REGISTRY
-import preprocessors.apycot.distutils
-from cubicweb.utils import json_dumps
-
-os.environ['HGRCPATH'] = os.devnull
-
-class FakeResponse(object):
-    def __init__(self, content):
-        self.content = content
-        self.reason = ''
-    def json(self):
-        return self.content
-
-class FakeCnxh(object):
-    def __init__(self, msgcfg, msgdep):
-        self.msgcfg = msgcfg
-        self.msgdep = msgdep
-        self.instance_url = 'http://gna'
-    def http_post(self, *args, **kwargs):
-        if 'vid' in kwargs and kwargs['vid'] == 'apycot.get_configuration':
-            return self.msgcfg
-        elif 'vid' in kwargs and kwargs['vid'] == 'apycot.get_dependencies':
-            return self.msgdep
-        else:
-            return []
-
-    def http_get(*args, **kwargs):
-        return [{'deps': 'a'}]
-
-    def rql(self, *args, **kw):
-        return FakeResponse(self.msgcfg)
-
-REGISTRY['preprocessor']['python_setup'] = preprocessors.apycot.distutils.DistutilsProcessor
-
-class FakeWriter(object):
-    def debug(self, msg, *args, **kwargs):
-        pass
-    def info(self, msg, *args, **kwargs):
-        pass
-    def fatal(self, msg, *args, **kwargs):
-        pass
-
-    def __init__(self, msgcfg, msgdep):
-        self.cnxh = FakeCnxh(msgcfg, msgdep)
-
-class FileCheckerTest(ApycotBaseTC):
-    def start_lgc_tests(self, cnx):
-        with self.admin_access.client_cnx() as cnx:
-            lgc = cnx.entity_from_eid(self.lgc)
-            lgce = cnx.entity_from_eid(self.lgce)
-            plan = lgc.start(lgce)
-            cnx.commit()
-            return plan.eid
-
-    def test_exec_status_change(self):
-        with self.admin_access.client_cnx() as cnx:
-            tceid = cnx.create_entity('TestConfig',
-                                      name=u'TC',
-                                      label=u'full python tests',
-                                      start_mode=u'manual',
-                                      check_config=u"""pylint_threshold=7
-    pycoverage_threshold=70
-    install=python_setup""").eid
-            peeid = cnx.create_entity('ProjectEnvironment',
-                                      name=u'gna',
-                                      reverse_use_environment=tceid,
-                                      local_repository=self.vcsrepo).eid
-
-            self.vcsrepo2 = cnx.create_entity('Repository', type=u'mercurial',
-                    source_url=u'file://' + unicode(self.datapath('project1')),
-                    reverse_local_repository=self.lgce)
-
-            pe_dep1 = cnx.create_entity('ProjectEnvironment',
-                                        name=u'regna',
-                                        reverse_use_environment=tceid,
-                                        local_repository=self.vcsrepo2)
-            cnx.commit()
-
-        with self.admin_access.web_request(environment=peeid) as req_env:
-            tc = req_env.entity_from_eid(tceid)
-            pe = req_env.entity_from_eid(peeid)
-            tconf = json.loads(req_env.view('apycot.get_configuration', rset=tc.as_rset()))
-            penv = json.loads(req_env.view('ejsonexport', rset=pe.as_rset()))[0]
-            penv_dep = json.loads(req_env.view('ejsonexport', rset=pe.as_rset()))
-            penv['title'] = 'gna'
-            planeid = self.start_lgc_tests(req_env)
-
-        with self.new_access('narval').client_cnx() as cnx:
-            plan = json.loads(json_dumps(cnx.entity_from_eid(planeid)))
-            test = atest.Test(plan, FakeWriter(tconf, penv_dep))
-            test.call_preprocessor('install', penv)
-
-
-if __name__ == '__main__':
-    unittest_main()
--- a/test/unittest_repositories.py	Thu Dec 11 22:18:16 2014 +0100
+++ b/test/unittest_repositories.py	Fri Nov 14 12:51:47 2014 +0100
@@ -27,9 +27,9 @@
 class HGRepositoryTC(testlib.TestCase):
     def test_co_path(self):
         vcsfile = VCSFile('mercurial', source_url=u'file://' + os.path.join(self.datadir, 'badsyntax'))
-        repo = HGRepository({'repository': vcsfile, 'path': 'common'})
+        repo = HGRepository({'repository': vcsfile, 'local_cache': 'common'})
         self.assertEqual(repo.co_path, 'badsyntax/common')
-        repo = HGRepository({'repository': vcsfile, 'path': 'common/sub'})
+        repo = HGRepository({'repository': vcsfile, 'local_cache': 'common/sub'})
         self.assertEqual(repo.co_path, 'badsyntax/common/sub')
 
 if __name__ == '__main__':
--- a/test/unittest_task.py	Thu Dec 11 22:18:16 2014 +0100
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,285 +0,0 @@
-#!/usr/bin/python
-
-import shutil
-import tarfile
-import sys
-import os
-from os.path import exists, join, abspath
-
-from logilab.common.testlib import TestCase, unittest_main, mock_object, within_tempdir
-
-# import this first will set import machinery on
-import cubicweb.devtools
-from cubes.apycot.testutils import MockTestWriter, MockRepository, MockVCSFile
-
-from apycotlib import SetupException
-from apycotlib import SUCCESS, FAILURE, PARTIAL, SKIPPED
-from apycotlib.atest import Test as BaseTest
-
-from unittest_checkers import input_path
-# manage temporary repo
-def setUpModule():
-    os.environ['APYCOT_ROOT'] = ''
-    mockvcsfile = MockVCSFile('subversion', path='/home/cvs')
-    global MOCKREPO, BADREPO, TPP
-    MOCKREPO = MockRepository(repository=mockvcsfile,
-                              path='soft/goodpkg',
-                              command='cp -R %s .' % input_path('goodpkg'))
-    BADREPO = MockRepository(repository=mockvcsfile,
-                             path='soft/goodpkg', command='false')
-    TPP = TouchTestPreprocessor()
-
-
-def tearDownModule():
-    del os.environ['APYCOT_ROOT']
-    for repo in ('badpkg2', 'goodpkg'):
-        path = input_path(repo)
-        if exists(path):
-            shutil.rmtree(path)
-
-def Test(tconfig, writer, preprocessors=None, checkers=None,
-         repo=None, environment=None):
-    pps = preprocessors
-    repo = repo or MOCKREPO
-    environment = environment or Environment(eid=tconfig['name'], apycot_preprocessors={})
-    texec = dict(configuration=tconfig, environment=environment,
-                 branch=environment['apycot_configuration'].get('branch'),
-                 options='')
-    test = BaseTest(texec, writer)
-    if pps is not None:
-        test.apycot_preprocessors = lambda x: pps
-    if checkers is not None:
-        test.checkers = checkers
-    if repo is not None:
-        test._repositories[environment['eid']] = repo
-    return test
-
-class TestConfig(dict):
-    def __init__(self, name, dependencies=(), environ=None, conf=None):
-        super(TestConfig, self).__init__()
-        self['name'] = self['eid'] = name
-        self['apycot_process_environment'] = environ or {'ZIGUOUIGOUI': 'YOOOO'}
-
-
-class Environment(dict):
-    def __init__(self, eid, apycot_preprocessors={},
-                 repository='default',
-                 conf=None):
-        if repository is 'default':
-            repository = dict(type='mercurial', source_url='http://bob.org/hg/toto/', path='')
-        super(Environment, self).__init__(
-                eid=eid, name='', vcs_path='',
-                apycot_configuration=conf or {},
-                apycot_process_environment={},
-                repository=repository)
-
-    def dc_title(self):
-        return self.name
-
-# mock objects ################################################################
-
-class CleanRaisePreprocessor:
-    id = 'clean_raise_preprocessor'
-    def match(self, name):
-        return 1
-
-    def run(self, test, path=None):
-        if path is None:
-            return 1
-        else:
-            return 0
-
-class SetupRaisePreprocessor:
-    id = 'setup_raise_preprocessor'
-    def match(self, name):
-        return 1
-
-    def run(self, test, path=None):
-        if path is None:
-            raise SetupException('in test_preprocessor.test_setup')
-        else:
-            raise SetupException('%s failed on %r' % (self.id, path))
-
-class TouchTestPreprocessor:
-    id = 'touch_preprocessor'
-    file = None
-    file2 = None
-
-    def run(self, test, path=None):
-        self.file = join(test.tmpdir, 'TestTC_pp')
-        self.file2 = join(test.tmpdir, 'TestTC2_pp')
-        f = open(self.file, 'w')
-        f.close()
-        f = open(self.file2, 'w')
-        f.close()
-
-class SimplePreprocessor(object):
-
-    id = 'simple_preprocessor'
-    def __init__(self):
-        self.processed    = {}
-
-    def run(self, test, path=None):
-        if path == None:
-            path = test.project_path()
-        self.processed.setdefault(path, 0)
-        self.processed[path] += 1
-
-class DummyTest(object):
-    need_preprocessor = None
-
-    def check(self, test, writer):
-        return SUCCESS
-    def check_options(self):
-        pass
-
-class SuccessTestChecker(DummyTest):
-    id = 'success_test_checker'
-    options = {}
-    need_preprocessor = 'install'
-    def check(self, test, writer):
-        return SUCCESS
-
-class FailureTestChecker(DummyTest):
-    id = 'failure_test_checker'
-    options = {}
-    def check(self, test, writer):
-        return FAILURE
-
-class ErrorTestChecker(DummyTest):
-    id = 'error_test_checker'
-    options = {}
-    def check(self, test, writer):
-        raise Exception('never succeed!')
-
-
-# real tests ##################################################################
-
-class TestTC(TestCase):
-
-    def test_branch(self):
-        test = Test(TestConfig('yo'),
-                    MockTestWriter(),
-                    environment=Environment('babar', conf={'branch': 'bob'}),
-                    checkers=[SuccessTestChecker()])
-        del test._repositories['babar'] # XXX clean up this mess
-        repo = test.apycot_repository()
-        self.assertEqual(repo.branch, 'bob')
-
-    def test_branch_deps_with_branch(self):
-        dep = Environment('babar', conf={'branch': 'Onk'})
-        test = Test(TestConfig('yo', dependencies=(dep, )),
-                    MockTestWriter(),
-                    checkers=[SuccessTestChecker()])
-        repo = test.apycot_repository(dep)
-        self.assertEqual(repo.branch, 'Onk')
-
-    def test_branch_deps_without_branch(self):
-        dep = Environment('babar')
-        test = Test(TestConfig('yo', dependencies=(dep, )),
-                    MockTestWriter(),
-                    checkers=[SuccessTestChecker()])
-        repo = test.apycot_repository(dep)
-        # default should be the branch name (as none is defined)
-        self.assertEqual(repo.branch, 'default')
-
-    def test_missing_repo(self):
-        dep = Environment('babar', repository=None)
-        test = Test(TestConfig('yo', dependencies=(dep,)),
-                    MockTestWriter(), {},
-                    checkers=[SuccessTestChecker()])
-        self.assertRaises(Exception, test.apycot_repository, dep)
-
-
-class EnvironmentTrackerMixinTC(TestCase):
-
-    def setUp(self):
-        self._PYTHONPATH = os.environ.get('PYTHONPATH', '')
-        os.environ['PYTHONPATH'] = ''
-        self.tracker = Test(TestConfig('yo', dependencies=(Environment('pypasax'),)),
-                            MockTestWriter())
-
-    def tearDown(self):
-        os.environ['PYTHONPATH'] = self._PYTHONPATH
-
-    def test_update_clean_env(self):
-        lc_all = os.environ.get('LC_ALL')
-        self.tracker.update_env('key', 'LC_ALL', 'XXXX')
-        self.assertEqual(os.environ['LC_ALL'], 'XXXX')
-        self.tracker.clean_env('key', 'LC_ALL')
-        self.assertEqual(os.environ.get('LC_ALL'), lc_all)
-
-        self.tracker.update_env('key', '__ENVIRONMENTTRACKERMIXINTC__', 'XXXX')
-        self.assertEqual(os.environ['__ENVIRONMENTTRACKERMIXINTC__'], 'XXXX')
-        self.tracker.clean_env('key', '__ENVIRONMENTTRACKERMIXINTC__')
-        self.assertRaises(KeyError, os.environ.__getitem__,
-                          '__ENVIRONMENTTRACKERMIXINTC__')
-
-    def test_nested(self):
-        lc_all = os.environ.get('LC_ALL')
-        self.tracker.update_env('key', 'LC_ALL', 'XXXX')
-        self.assertEqual(os.environ['LC_ALL'], 'XXXX')
-        self.tracker.update_env('key2', 'LC_ALL', 'YYYY')
-        self.assertEqual(os.environ['LC_ALL'], 'YYYY')
-        self.tracker.clean_env('key2', 'LC_ALL')
-        self.assertEqual(os.environ['LC_ALL'], 'XXXX')
-        self.tracker.clean_env('key', 'LC_ALL')
-        self.assertEqual(os.environ.get('LC_ALL'), lc_all)
-
-    def test_update_clean_env_sep(self):
-        path = os.environ['PATH']
-        self.tracker.update_env('key', 'PATH', '/mybin', ':')
-        self.assertEqual(os.environ['PATH'], '/mybin:' + path)
-        self.tracker.clean_env('key', 'PATH')
-        self.assertEqual(os.environ['PATH'], path)
-
-    def test_nested_sep(self):
-        path = os.environ['PATH']
-        self.tracker.update_env('key', 'PATH', '/mybin', ':')
-        if path:
-            self.assertEqual(os.environ['PATH'], '/mybin:' + path)
-        else:
-            self.assertEqual(os.environ['PATH'], '/mybin')
-        self.tracker.update_env('key2', 'PATH', '/myotherbin', ':')
-        if path:
-            self.assertEqual(os.environ['PATH'], '/myotherbin:/mybin:' + path)
-        else:
-            self.assertEqual(os.environ['PATH'], '/myotherbin:/mybin')
-        self.tracker.clean_env('key2', 'PATH')
-        if path:
-            self.assertEqual(os.environ['PATH'], '/mybin:' + path)
-        else:
-            self.assertEqual(os.environ['PATH'], '/mybin')
-        self.tracker.clean_env('key', 'PATH')
-        self.assertEqual(os.environ['PATH'], path)
-
-    def test_python_path_sync(self):
-        self.tracker.update_env('key', 'PYTHONPATH', '/mylib', ':')
-        self.assertEqual(os.environ['PYTHONPATH'], '/mylib')
-        self.assertEqual(sys.path[0], '/mylib')
-        self.tracker.update_env('key2', 'PYTHONPATH', '/otherlib', ':')
-        self.assertEqual(os.environ['PYTHONPATH'], '/otherlib:/mylib')
-        self.assertEqual(sys.path[0], '/otherlib')
-        self.tracker.clean_env('key2', 'PYTHONPATH')
-        self.assertEqual(os.environ['PYTHONPATH'], '/mylib')
-        self.assertNotEqual(sys.path[0], '/otherlib')
-        self.tracker.clean_env('key', 'PYTHONPATH')
-        self.assertEqual(os.environ['PYTHONPATH'], '')
-        self.assertNotEqual(sys.path[0], '/otherlib')
-
-    def test_update_undefined_env(self):
-
-        var = 'XNZOUACONFVESUHFJGSLKJ'
-        while os.environ.get(var) is not None:
-            var = ''.join(chr(randint(ord('A'), ord('Z') +1))
-                for cnt in xrange(randint(10, 20)))
-
-        self.tracker.update_env('key', var, 'to be or not to be', ':')
-        self.assertMultiLineEqual(os.environ.get(var),  'to be or not to be')
-        self.tracker.clean_env('key', var)
-        self.assertEqual(os.environ.get(var), None)
-
-
-
-if __name__ == '__main__':
-    unittest_main()
--- a/testutils.py	Thu Dec 11 22:18:16 2014 +0100
+++ b/testutils.py	Fri Nov 14 12:51:47 2014 +0100
@@ -6,13 +6,15 @@
 
 from logilab.common.testlib import mock_object
 from logilab.common.shellutils import unzip
+from logilab.common.decorators import classproperty
 
 from cubicweb.devtools import BASE_URL
 from cubicweb.devtools.testlib import CubicWebTC
 
-from cubes.vcsfile.testutils import init_vcsrepo
+from cubes.vcsfile import bridge
+from cubes.vcsfile.testutils import setup_repos, HGRCMixin
+
 from cubes.narval.testutils import NarvalBaseTC
-from cubes.apycot.recipes import quick_script
 
 try:
     import apycotlib
@@ -35,12 +37,6 @@
 
 from apycotlib.writer import CheckDataWriter, BaseDataWriter
 
-def setup_repos(datadir, *reponames):
-    for repo in reponames:
-        repopath = join(datadir, repo)
-        if exists(repopath):
-            shutil.rmtree(repopath)
-        unzip(join(datadir, '%s.zip') % repo, datadir)
 
 class DummyStack(object):
 
@@ -154,43 +150,62 @@
     def plan(self, eid):
         pass
 
-class ApycotBaseTC(NarvalBaseTC):
+class ApycotBaseTC(HGRCMixin, NarvalBaseTC):
+    _repo_path = ()
+    repo_import_revision_content = True
+
+    @classproperty
+    def test_db_id(cls):
+        if cls._repo_path is None:
+            return None
+        ids = list(cls._repo_path)
+        if not cls.repo_import_revision_content:
+            ids.append('nocontent')
+        return '-'.join(ids)
 
-    recipescript = quick_script
+    @classmethod
+    def pre_setup_database(cls, cnx, config):
+        super(ApycotBaseTC, cls).pre_setup_database(cnx, config)
+        setup_repos(*[join(cls.datadir, path) for path in cls._repo_path])
+        pyp_tc = cnx.create_entity('TestConfig', name=u'PYTHONPACKAGE',
+                                   check_config=u'python_lint_treshold=7\n'
+                                   'python_lint_ignore=thirdparty\n'
+                                   'python_test_coverage_treshold=70\n',
+                                   check_environment=u'NO_SETUPTOOLS=1\nDISPLAY=:1.0')
+        quickrecipe = cnx.find('Recipe', name='apycot.quick').one()
+        setuprecipe = cnx.find('Recipe', name='apycot.setup.distutils').one()
+        for path in cls._repo_path:
+            repo = cnx.create_entity(
+                'Repository', type=u'mercurial',
+                source_url=u'file://' + join(cls.datadir, path), title=path,
+                import_revision_content=True)
+            pe = cnx.create_entity(
+                'ProjectEnvironment', name=u'pe_%s'%path,
+                check_config=u'env-option=value',
+                check_environment=u'SETUPTOOLS=1\nDISPLAY=:2.0',
+                setup_recipe=setuprecipe)
+            cls.add_test_config(cnx, u'tc_%s'%path, env=pe,
+                                group=pyp_tc, use_recipe=quickrecipe)
+        cnx.commit()
+
+    def setUp(self):
+        setup_repos(*[join(self.datadir, path) for path in self._repo_path])
+        super(ApycotBaseTC, self).setUp()
+        self.refresh()
+
+    def tearDown(self):
+        super(ApycotBaseTC, self).tearDown()
+        for path in self._repo_path:
+            shutil.rmtree(join(self.datadir, path), ignore_errors=True)
 
     def setup_database(self):
         """ self.repo: used to get the session to connect to cw
             self.vcsrepo: new entity
         """
-        with self.admin_access.repo_cnx() as cnx:
-            self.lgce = cnx.create_entity(
-                'ProjectEnvironment', name=u'lgce',
-                check_config=u'install=python_setup\nenv-option=value',
-                check_environment=u'SETUPTOOLS=1\nDISPLAY=:2.0'
-                ).eid
-            self.vcsrepo = cnx.create_entity('Repository', type=u'mercurial',
-                                             source_url=u'file://' + unicode(self.datapath('project')),
-                                             reverse_local_repository=self.lgce).eid
-            self.pyp = cnx.create_entity('TestConfig', name=u'PYTHONPACKAGE',
-                                         check_config=u'python_lint_treshold=7\n'
-                                         'python_lint_ignore=thirdparty\n'
-                                         'python_test_coverage_treshold=70\n',
-                                         check_environment=u'NO_SETUPTOOLS=1\nDISPLAY=:1.0').eid
-            self.recipe = cnx.execute('Recipe X WHERE X name "apycot.recipe.quick"')[0][0]
-            # reset vcsrepo (using the session )
-            init_vcsrepo(self.repo)
-            # reset recipe content
-            cnx.execute('SET X script %(script)s WHERE X eid %(recipe)s',
-                        {'recipe': self.recipe,
-                         'script': self.recipescript})
-            self.lgc = self.add_test_config(cnx, u'lgc', env=self.lgce,
-                                            group=self.pyp,
-                                            use_recipe=self.recipe).eid
-            cnx.commit()
+        self.repo.threaded_task = lambda func: func() # XXX move to cw
 
-            self.repo.threaded_task = lambda func: func() # XXX move to cw
-
-    def add_test_config(self, cnx, name,
+    @classmethod
+    def add_test_config(cls, cnx, name,
                         check_config=u'python_lint_treshold=8\npouet=5',
                         env=None, group=None, **kwargs):
         """add a TestConfig instance"""
@@ -204,9 +219,20 @@
     def dumb_execution(self, cnx, ex, check_defs, setend=True):
         """add a TestExecution instance"""
         for name, status in check_defs:
-            cr = cnx.create_entity('CheckResult', name=unicode(name), status=unicode(status))
+            cr = cnx.create_entity('CheckResult', name=unicode(name),
+                                   status=unicode(status))
             cnx.execute('SET X during_execution Y WHERE X eid %(x)s, Y eid %(e)s',
                         {'x': cr.eid, 'e': ex.eid})
         if setend:
             cnx.execute('SET X status "success" '
                         'WHERE X eid %(x)s', {'x': ex.eid})
+    def hgrepo(self, reponame):
+        with self.admin_access.repo_cnx() as cnx:
+            repopath = cnx.find('Repository', title=reponame).one().localcachepath
+        return hgopen(repopath)
+
+    def refresh(self):
+        with self.repo.internal_cnx() as cnx:
+            bridge.import_content(cnx, commitevery=1, raise_on_error=True)
+
+