[apycotlib] port to narval 4.2 draft
authorDavid Douard <david.douard@logilab.fr>
Sun, 26 Oct 2014 18:30:36 +0100
changeset 1736 72ec6c7e956c
parent 1735 ef4cbeb91ec6
child 1737 52222f1b82d4
[apycotlib] port to narval 4.2 The connection handler is now a CWProxy (cwclientlib), and there is no CreateSubEntity view any more since we can now create File entity using cwclientlib and rqlcontroller.
_apycotlib/atest.py
_apycotlib/narvalactions.py
_apycotlib/writer.py
test/test_functional.py
test/unittest_hooks.py
test/unittest_preprocessors.py
testutils.py
--- a/_apycotlib/atest.py	Tue Oct 14 08:49:14 2014 +0200
+++ b/_apycotlib/atest.py	Sun Oct 26 18:30:36 2014 +0100
@@ -113,10 +113,9 @@
         try:
             return self._configs[pe['eid']]
         except KeyError:
-            tconfig_url = self.writer.cnxh.instance_url + str(self.tconfig['eid'])
-            config = self.writer.cnxh.http_get(tconfig_url,
-                                                vid='apycot.get_configuration',
-                                                environment=pe['eid'])[0]
+            config = self.writer.cnxh.rql('Any E WHERE E eid %s' % self.tconfig['eid'],
+                                          vid='apycot.get_configuration',
+                                          environment=pe['eid']).json()[0]
             self._configs[pe['eid']] = config
             self._substitute(pe, config)
             return config
--- a/_apycotlib/narvalactions.py	Tue Oct 14 08:49:14 2014 +0200
+++ b/_apycotlib/narvalactions.py	Sun Oct 26 18:30:36 2014 +0100
@@ -17,8 +17,8 @@
         self.test.clean()
 
 def install_environment(test):
-    texec_url = test.writer.cnxh.instance_url + str(test.texec['eid'])
-    data = test.writer.cnxh.http_get(texec_url, vid='apycot.get_dependencies')
+    data = test.writer.cnxh.rql('Any E WHERE E eid %s' % test.texec['eid'],
+                                vid='apycot.get_dependencies').json()
     for dep in data[0]:
         test.checkout(dep)
         test.call_preprocessor('install', dep)
--- a/_apycotlib/writer.py	Tue Oct 14 08:49:14 2014 +0200
+++ b/_apycotlib/writer.py	Sun Oct 26 18:30:36 2014 +0100
@@ -15,8 +15,12 @@
 import traceback
 from datetime import datetime
 from StringIO import StringIO
+from io import BytesIO
 from threading import RLock
 from logilab.mtconverter import xml_escape
+from logilab.common.date import ustrftime
+
+from cwclientlib import builders
 
 REVERSE_SEVERITIES = {
     logging.DEBUG :   u'DEBUG',
@@ -111,7 +115,6 @@
         self.cnxh = cnxh
         # eid of the execution entity
         self.target_eid = target_eid
-        self._url = self.cnxh.instance_url + str(target_eid)
         self._log_file_eid = None
         self._logs = []
         self._logs_sent = 0
@@ -122,9 +125,6 @@
     def end(self):
         pass
 
-    def set_exec_status(self, status):
-        self.cnxh.http_post(self._url, vid='set_attributes', status=status)
-
     def execution_info(self, *args, **kwargs):
         msg = self._msg_info(*args, **kwargs)[-1]
         if isinstance(msg, unicode):
@@ -139,28 +139,34 @@
 
     def raw(self, name, value, type=None):
         """give some raw data"""
-        self.cnxh.http_post(self._url, vid='create_subentity',
-                             __cwetype__='CheckResultInfo',
-                             __cwrel__='for_check',
-                             label=self._unicode(name),
-                             value=self._unicode(value),
-                             type=type and unicode(type))
+        self.cnxh.rqlio([builders.create_entity('CheckResultInfo',
+                                                label=self._unicode(name),
+                                                value=self._unicode(value),
+                                                type=type and unicode(type)),
+                         ('SET CRI for_check TE WHERE CRI eid %(cri)s, TE eid %(te)s',
+                          {'cri': '__r0', 'te': self.target_eid})])
 
     def refresh_log(self):
         log = self._logs
+        if not self._log_file_eid:
+            data = self.cnxh.rqlio([('INSERT File F: F data %(content)s, '
+                                     'F data_name %(name)s, '
+                                     'F data_encoding %(coding)s, '
+                                     'F data_format %(fmt)s, '
+                                     'TE log_file F WHERE TE eid %(eid)s',
+                                     {'name': u'log_file.txt',
+                                      'fmt': u'text/plain',
+                                      'coding': 'utf-8',
+                                      'content': BytesIO(''),
+                                      'eid': self.target_eid,
+                                      })])
+            self._log_file_eid = data.json()[0][0][0]
+
         if self._logs_sent < len(log):
-            files = {'data': ('dummy', u''.join(log[self._logs_sent:]) )}
-            if not self._log_file_eid:
-                data = self.cnxh.http_post(self._url, vid='create_subentity',
-                                            __cwetype__='File',
-                                            __cwrel__='reverse_log_file',
-                                            data_name=u'log_file.txt',
-                                            data_encoding='utf-8')[0]
-                self._log_file_eid = data['eid']
-            self.cnxh.http_post(url=self.cnxh.instance_url + 'narval-file-append',
-                                files=files,
-                                eid=self._log_file_eid)
-
+            logcontent = u''.join(log[self._logs_sent:]).encode('utf-8')
+            files = {'to_append': logcontent}
+            self.cnxh.http_post('narval-file-append',
+                                files=files, eid=self._log_file_eid)
             self._logs_sent = len(log)
 
 
@@ -176,12 +182,13 @@
             crname = name
         else:
             crname = getattr(checker, 'id', checker) # may be the checked id
-        data = self.cnxh.http_post(self._url, vid='create_subentity',
-                                    __cwetype__='CheckResult',
-                                    __cwrel__='during_execution',
-                                    name=self._unicode(crname), status=u'processing',
-                                    starttime=datetime.now())
-        self._url = self.cnxh.instance_url + str(data[0]['eid'])
+        eid = self.cnxh.rqlio([builders.create_entity('CheckResult',
+                                                      name=self._unicode(crname), status=u'processing',
+                                                      starttime=ustrftime(datetime.now(), '%Y-%m-%d %H:%M:%S')),
+                               ('SET CR during_execution TE WHERE CR eid %(cr)s, TE eid %(te)s',
+                                {'cr': '__r0', 'te': self.target_eid})]).json()[0][0][0]
+        self.target_eid = eid
+        self._log_file_eid = None
         if hasattr(checker, 'options'):
             options = ['%s=%s' % (k, v) for k, v in checker.options.iteritems()
                        if k in checker.options_def
@@ -193,9 +200,10 @@
     def end(self, status):
         """Register the given checker as closed with status <status>"""
         self.refresh_log()
-        self.cnxh.http_post(self._url, vid='set_attributes',
-                             status=self._unicode(status),
-                             endtime=datetime.now(),)
+        self.cnxh.rqlio([('SET CR status %(status)s, CR endtime %(endtime)s WHERE CR eid %(eid)s',
+                          {'status': self._unicode(status),
+                           'endtime': ustrftime(datetime.now(), '%Y-%m-%d %H:%M:%S'),
+                           'eid': self.target_eid})])
 
 
 
@@ -218,17 +226,16 @@
         if changeset is not None:
             self.raw(repr(vcsrepo), changeset, 'revision')
 
-    def start(self):
-        self.set_exec_status(u'set up')
-
     def end(self, status, archivedir=None):
         """mark the current test as closed (with status <status>) and upload
         archive if requested."""
 
         self.cnxh.debug('End test with status %s' % status)
         self.refresh_log()
-        self.cnxh.http_post(self._url, vid='set_attributes',
-                            status=self._unicode(status))
+        self.cnxh.rqlio([('SET CR status %(status)s WHERE CR eid %(eid)s',
+                          {'status': self._unicode(status),
+                           'eid': self.target_eid})])
+        
         if archivedir:
             self.cnxh.debug('Archive the apycot temp directory')
             archive = make_archive_name(self.cnxh.instance_id, self.target_eid)
--- a/test/test_functional.py	Tue Oct 14 08:49:14 2014 +0200
+++ b/test/test_functional.py	Sun Oct 26 18:30:36 2014 +0100
@@ -13,25 +13,23 @@
 DATA = osp.join(osp.dirname(__file__), 'data')
 os.environ['HGRCPATH'] = os.devnull
 
-import narvalbot
-narvalbot._CW_SOURCES_FILE = osp.join(DATA, 'narval-cw-sources.ini')
-
+class ApycotTC(HGRCMixin, utils.ApycotBaseTC):
+    _repo_path = u'project'
 
-for repo in ('project', ):
-    repopath = osp.join(DATA, repo)
-    if osp.exists(repopath):
-        rmtree(repopath)
-    unzip(osp.join(DATA, '%s.zip') % repo, DATA)
+    def setUp(self):
+        utils.setup_repos(DATA, self._repo_path)
+        super(ApycotTC, self).setUp()
 
-class ApycotTC(HGRCMixin, utils.ApycotBaseTC):
 
     def test_quick_recipe(self):
         with self.admin_access.client_cnx() as cnx:
             lgc = cnx.entity_from_eid(self.lgc)
             lgce = cnx.entity_from_eid(self.lgce)
-            te = lgc.start(lgce)
+            te = lgc.start(lgce).eid
             cnx.commit()
-            self.run_plan(te)
+        self.run_plan(te)
+        with self.admin_access.client_cnx() as cnx:
+            te = cnx.entity_from_eid(te)
             self.assertEqual(dict((checker.name, checker.status) for checker in te.checkers),
                              {'pyunit': 'nodata'})
 
@@ -43,9 +41,11 @@
             recipe.cw_set(script=full_script)
             tc = self.add_test_config(cnx, u'full config', env=self.lgce, group=self.pyp,
                                       use_recipe=recipe)
-            te = tc.start(lgce)
+            te = tc.start(lgce).eid
             cnx.commit()
-            self.run_plan(te)
+        self.run_plan(te)
+        with self.admin_access.client_cnx() as cnx:
+            te = cnx.entity_from_eid(te)
             exp = {u'pycoverage': u'error', u'pyunit': u'nodata'}
             try:
                 # do only check pylint tests where executed if pylint
--- a/test/unittest_hooks.py	Tue Oct 14 08:49:14 2014 +0200
+++ b/test/unittest_hooks.py	Sun Oct 26 18:30:36 2014 +0100
@@ -83,7 +83,7 @@
 Imported changes occured between XXXX/XX/XX XX:XX and XXXX/XX/XX XX:XX:
 * no change found in known repositories
 
-URL: http://testing.fr/cubicweb/testexecution/XXXX''')
+URL: http://XXX.X.X.X:XXXX/testexecution/XXXX''')
 
 
     def test_exec_one_status_change(self):
@@ -117,7 +117,7 @@
 Imported changes occured between XXXX/XX/XX XX:XX and XXXX/XX/XX XX:XX:
 * no change found in known repositories
 
-URL: http://testing.fr/cubicweb/testexecution/XXXX''')
+URL: http://XXX.X.X.X:XXXX/testexecution/XXXX''')
 
     def test_nosy_list_propagation(self):
         planeid = self.start_lgc_tests()
@@ -165,34 +165,39 @@
             rql +=', TC computed_start_mode "%s"' % period
         return set(tuple(tc) for tc in cnx.execute(rql))
 
-    def get_te_for_tc_name(self, cnx, eid):
+    def get_te_for_tc(self, cnx, eid):
         rql = 'Any TE where TE using_config TC, TC eid %(eid)s'
         return cnx.execute(rql, {'eid': eid})
 
-    def launch_all_tests(self, cnx, period):
+    def launch_all_tests(self, period):
         """ launch all tests in the correct state and and make sure no
         TestExecutions are left in status waiting execution. Also fill-in the
         revision which should normally be filled by StartTestOp.
         """
 
-        start_period_tests(cnx, period)
-        cnx.commit() # put new te in base
-        ## make sure they are not waiting execution and REV is filled
-        for te, rev in cnx.execute('DISTINCT Any TE, REV WHERE TE status %(st)s, '
-                                    'TE using_config TC, '
-                                    'TE branch BR, '
-                                    'TC use_environment PE, '
-                                    'PE local_repository R, '
-                                    'REV from_repository R, '
-                                    'REV branch BR, '
-                                    'NOT REV parent_revision REV2',
-                                    {'st': 'waiting execution'}):
-            te_e = cnx.entity_from_eid(te)
-            rev_e = cnx.entity_from_eid(rev)
-            te_e.cw_adapt_to('IWorkflowable').fire_transition('start')
-            te_e.cw_set(status=u'running')
-            te_e.cw_set(using_revision=rev)
-        cnx.commit() # change new te status
+        with self.admin_access.client_cnx() as cnx:
+            start_period_tests(cnx, period)
+            cnx.commit() # put new te in base
+
+        with self.new_access('narval').client_cnx() as cnx:
+            ## make sure they are not waiting execution and REV is filled
+            for (te, rev) in cnx.execute('DISTINCT Any TE, REV WHERE TE in_state ST, '
+                                         'ST name "ready", '
+                                         'TE using_config TC, '
+                                         'TE branch BR, '
+                                         'TC use_environment PE, '
+                                         'PE local_repository R, '
+                                         'REV from_repository R, '
+                                         'REV branch BR, '
+                                         'NOT REV parent_revision REV2'):
+                te_e = cnx.entity_from_eid(te)
+                te_e.cw_adapt_to('IWorkflowable').fire_transition('start')
+                rev_e = cnx.entity_from_eid(rev)
+                # force status and revision since we do not have a narvabot eating
+                # our TestExecutions here
+                te_e.cw_set(status=u'running')
+                te_e.cw_set(using_revision=rev)
+            cnx.commit() # change new te status
 
     def test_new_vc_trigger(self):
         """ check the on new revision start mode. Run all testconfigs, add new
@@ -237,28 +242,26 @@
                                   ('lgce', 'lgc3', None),
                                   ('lgce', 'lgc4', None),
                                   ('lgce2', 'lgc5', 1)]))
-            ## run everything 
-
-            self.launch_all_tests(cnx, 'on new revision')
+        ## create TestExecution entities
+        self.launch_all_tests('on new revision')
 
-            nb_tc_lgc_before = len(self.get_te_for_tc_name(cnx, self.lgc))
-            nb_tc_lgc3_before = len(self.get_te_for_tc_name(cnx, lgc3.eid))
-            nb_tc_lgc4_before = len(self.get_te_for_tc_name(cnx, lgc4.eid))
-            nb_tc_lgc5_before = len(self.get_te_for_tc_name(cnx, lgc5.eid))
-            ##check you do not add anything on rerun (no new revision)
-            self.launch_all_tests(cnx, 'on new revision')
+        with self.admin_access.client_cnx() as cnx:
+            nb_tc_lgc_before = len(self.get_te_for_tc(cnx, self.lgc))
+            nb_tc_lgc3_before = len(self.get_te_for_tc(cnx, lgc3.eid))
+            nb_tc_lgc4_before = len(self.get_te_for_tc(cnx, lgc4.eid))
+            nb_tc_lgc5_before = len(self.get_te_for_tc(cnx, lgc5.eid))
+            ## check you do not add anything on rerun (no new revision)
+
+        self.launch_all_tests('on new revision')
+        with self.admin_access.client_cnx() as cnx:
             self.assertEqual(nb_tc_lgc_before,
-                             len(self.get_te_for_tc_name(cnx, self.lgc)))
+                             len(self.get_te_for_tc(cnx, self.lgc)))
             self.assertEqual(nb_tc_lgc3_before,
-                             len(self.get_te_for_tc_name(cnx, lgc3.eid)))
+                             len(self.get_te_for_tc(cnx, lgc3.eid)))
             self.assertEqual(nb_tc_lgc4_before,
-                             len(self.get_te_for_tc_name(cnx, lgc4.eid)))
+                             len(self.get_te_for_tc(cnx, lgc4.eid)))
             self.assertEqual(nb_tc_lgc5_before,
-                             len(self.get_te_for_tc_name(cnx, lgc5.eid)))
-            ## add a new revision in stable
-            #r.vcs_add(u'dir1', u'tutu.png', Binary('data'), branch=u'stable')
-            cnx.commit()
-
+                             len(self.get_te_for_tc(cnx, lgc5.eid)))
         r = self.repo
         os.system('echo data > %s/tutu1.png' % r.path)
         os.system('hg -R %s branch stable' % r.path)
@@ -266,40 +269,40 @@
         os.system('hg -R %s commit -m rien -u narval' % r.path)
         with r.internal_cnx() as cnx:
             bridge.import_content(cnx, commitevery=1, raise_on_error=True)
+            cnx.commit()
 
+        self.launch_all_tests('on new revision')
         with self.admin_access.client_cnx() as cnx:
-            ## now it should add te for the tc linked to lgce and running on new 
+            ## now it should add te for the tc linked to lgce and running on new
             ## revision and not the others
-            self.launch_all_tests(cnx, 'on new revision')
             self.assertEqual(nb_tc_lgc_before + 1,
-                             len(self.get_te_for_tc_name(cnx, self.lgc)))
+                             len(self.get_te_for_tc(cnx, self.lgc)))
             self.assertEqual(nb_tc_lgc3_before + 1,
-                             len(self.get_te_for_tc_name(cnx, lgc3.eid)))
+                             len(self.get_te_for_tc(cnx, lgc3.eid)))
             self.assertEqual(nb_tc_lgc4_before,
-                              len(self.get_te_for_tc_name(cnx, lgc4.eid)))
+                              len(self.get_te_for_tc(cnx, lgc4.eid)))
             self.assertEqual(nb_tc_lgc5_before,
-                              len(self.get_te_for_tc_name(cnx, lgc5.eid)))
+                              len(self.get_te_for_tc(cnx, lgc5.eid)))
 
-            ## add a new revision in default
-            #r.vcs_add(u'dir1', u'tutu1.png', Binary('data'))
-            cnx.commit()
         self.assertEqual(0, os.system('echo data > %s/tutu2.png' % r.path))
-        self.assertEqual(0,os.system('hg -R %s up default' % r.path))
-        self.assertEqual(0,os.system('hg -R %s add %s/tutu2.png' % (r.path, r.path)))
-        self.assertEqual(0,os.system('hg -R %s commit -m rien -u narval' % r.path))
+        self.assertEqual(0, os.system('hg -R %s up default' % r.path))
+        self.assertEqual(0, os.system('hg -R %s add %s/tutu2.png' % (r.path, r.path)))
+        self.assertEqual(0, os.system('hg -R %s commit -m rien -u narval' % r.path))
         with r.internal_cnx() as cnx:
             bridge.import_content(cnx, commitevery=1, raise_on_error=True)
+            cnx.commit()
+
         ## it should add te to the tc on default (lgc and lgc4)
+        self.launch_all_tests('on new revision')
         with self.admin_access.client_cnx() as cnx:
-            self.launch_all_tests(cnx, 'on new revision')
             self.assertEqual(nb_tc_lgc_before + 2,
-                             len(self.get_te_for_tc_name(cnx, self.lgc)))
+                             len(self.get_te_for_tc(cnx, self.lgc)))
             self.assertEqual(nb_tc_lgc3_before + 1,
-                             len(self.get_te_for_tc_name(cnx, lgc3.eid)))
+                             len(self.get_te_for_tc(cnx, lgc3.eid)))
             self.assertEqual(nb_tc_lgc4_before + 1,
-                             len(self.get_te_for_tc_name(cnx, lgc4.eid)))
+                             len(self.get_te_for_tc(cnx, lgc4.eid)))
             self.assertEqual(nb_tc_lgc5_before,
-                             len(self.get_te_for_tc_name(cnx, lgc5.eid)))
+                             len(self.get_te_for_tc(cnx, lgc5.eid)))
 
     def test_datetime_trigger(self):
         """test the registering of TC depending on their start_mode
@@ -338,7 +341,7 @@
                                    ('lgce2', 'lgc3', 1)))
                               )
             cnx.commit()
-            rset_before_lgc = self.get_te_for_tc_name(cnx, self.lgc)
+            rset_before_lgc = self.get_te_for_tc(cnx, self.lgc)
             te = lgc3.start(lgce2, branch=u'default')
             cnx.commit()
             ###not really useful as the method directly check the existence of a
@@ -348,11 +351,12 @@
                               set((('lgce', 'lgc', None),
                                    ('lgce2', 'lgc3', 1)))
                               )
-            rset_before_lgc = self.get_te_for_tc_name(cnx, self.lgc)
-            rset_before_lgc3 = self.get_te_for_tc_name(cnx, lgc3.eid)
-            self.launch_all_tests(cnx, 'hourly')
-            rset_after_lgc = self.get_te_for_tc_name(cnx, self.lgc)
-            rset_after_lgc3 = self.get_te_for_tc_name(cnx, lgc3.eid)
+            rset_before_lgc = self.get_te_for_tc(cnx, self.lgc)
+            rset_before_lgc3 = self.get_te_for_tc(cnx, lgc3.eid)
+        self.launch_all_tests('hourly')
+        with self.admin_access.client_cnx() as cnx:
+            rset_after_lgc = self.get_te_for_tc(cnx, self.lgc)
+            rset_after_lgc3 = self.get_te_for_tc(cnx, lgc3.eid)
             ### check there is one new TestConfig for lgc and the same number for lgc3
             #(alredy launched)
             self.assertEqual(len(rset_after_lgc3), len(rset_before_lgc3))
--- a/test/unittest_preprocessors.py	Tue Oct 14 08:49:14 2014 +0200
+++ b/test/unittest_preprocessors.py	Sun Oct 26 18:30:36 2014 +0100
@@ -12,6 +12,13 @@
 
 os.environ['HGRCPATH'] = os.devnull
 
+class FakeResponse(object):
+    def __init__(self, content):
+        self.content = content
+        self.reason = ''
+    def json(self):
+        return self.content
+
 class FakeCnxh(object):
     def __init__(self, msgcfg, msgdep):
         self.msgcfg = msgcfg
@@ -28,6 +35,9 @@
     def http_get(*args, **kwargs):
         return [{'deps': 'a'}]
 
+    def rql(self, *args, **kw):
+        return FakeResponse(self.msgcfg)
+
 REGISTRY['preprocessor']['python_setup'] = preprocessors.apycot.distutils.DistutilsProcessor
 
 class FakeWriter(object):
--- a/testutils.py	Tue Oct 14 08:49:14 2014 +0200
+++ b/testutils.py	Sun Oct 26 18:30:36 2014 +0100
@@ -2,9 +2,10 @@
 import shutil
 import os
 from datetime import datetime
-from os.path import join, dirname, abspath
+from os.path import join, dirname, abspath, exists
 
 from logilab.common.testlib import mock_object
+from logilab.common.shellutils import unzip
 
 from cubicweb.devtools import BASE_URL
 from cubicweb.devtools.testlib import CubicWebTC
@@ -25,10 +26,21 @@
 else:
     from cubes.narval.__pkginfo__ import NARVAL_DIR
     PLUGINSDIR = join(narvalbot.INSTALL_PREFIX, NARVAL_DIR)
+
 sys.path.append(PLUGINSDIR)
+PPATH = [PLUGINSDIR]
+if 'PYTHONPATH' in os.environ:
+    PPATH.append(os.environ['PYTHONPATH'])
+os.environ['PYTHONPATH'] = ':'.join(PPATH)
 
 from apycotlib.writer import CheckDataWriter, BaseDataWriter
 
+def setup_repos(datadir, *reponames):
+    for repo in reponames:
+        repopath = join(datadir, repo)
+        if exists(repopath):
+            shutil.rmtree(repopath)
+        unzip(join(datadir, '%s.zip') % repo, datadir)
 
 class DummyStack(object):
 
@@ -93,9 +105,6 @@
         self.checkers = []
         self._apycot_config = {}
 
-    def project_path(self, subpath=False):
-        return self.repo.co_path
-
     @property
     def tconfig(self):
         return mock_object(testconfig={}, name='bob', subpath=None)
@@ -175,7 +184,9 @@
             cnx.execute('SET X script %(script)s WHERE X eid %(recipe)s',
                         {'recipe': self.recipe,
                          'script': self.recipescript})
-            self.lgc = self.add_test_config(cnx, u'lgc', env=self.lgce, group=self.pyp, use_recipe=self.recipe).eid
+            self.lgc = self.add_test_config(cnx, u'lgc', env=self.lgce,
+                                            group=self.pyp,
+                                            use_recipe=self.recipe).eid
             cnx.commit()
 
             self.repo.threaded_task = lambda func: func() # XXX move to cw