[entities] Do not push already pushed resources
authorDenis Laxalde <denis.laxalde@logilab.fr>
Fri, 21 Nov 2014 17:15:40 +0100
changeset 16 4d8fea13570d
parent 15 f0c22bd9116d
child 17 7c8af9dda7d9
[entities] Do not push already pushed resources Look for existing resources on CKAN side and compare with their url (download url).
entities.py
test/unittest_hooks.py
--- a/entities.py	Fri Nov 21 16:33:31 2014 +0100
+++ b/entities.py	Fri Nov 21 17:15:40 2014 +0100
@@ -64,6 +64,14 @@
         # XXX CKAN uses Markdown syntax for notes field.
         return self.entity.printable_value('description', format='text/plain')
 
+    def ckan_package_resources(self):
+        """Return the list of resources URL for dataset"""
+        data = {'id': self.entity.ckan_dataset_id}
+        res = ckan_post(self._cw.vreg.config, 'package_show', data)
+        if res:
+            return [d['url'] for d in res['resources']]
+        return []
+
     def ckan_get_organization_id(self, orgname):
         """Retrieve the ID of an organization given its name"""
         data = {'organizations': [orgname],
@@ -122,6 +130,7 @@
 
     def ckan_resources(self):
         """Yield dicts of CKAN dataset resource info"""
+        resources_url = self.ckan_package_resources()
         for entity in self.dataset_resources():
             adapted = entity.cw_adapt_to('IDownloadable')
             if adapted is None:
@@ -129,7 +138,12 @@
                     'invalid resource %r, could not adapt to IDownloadable',
                     entity)
                 continue
-            yield {'url': adapted.download_url(),
+            url = adapted.download_url()
+            if url in resources_url:
+                self.info('skipping resource %s and it seems to be already '
+                          'on CKAN side', entity)
+                continue
+            yield {'url': url,
                    'name': adapted.download_file_name(),
                    'mimetype': adapted.download_content_type(),
                    'created': str(entity.creation_date),
--- a/test/unittest_hooks.py	Fri Nov 21 16:33:31 2014 +0100
+++ b/test/unittest_hooks.py	Fri Nov 21 17:15:40 2014 +0100
@@ -89,6 +89,14 @@
         r0 = resources[0]
         iresource = resource.cw_adapt_to('IDownloadable')
         self.assertEqual(r0['url'], iresource.download_url())
+        # Update the entity (should trigger another push to CKAN, but no new
+        # resource).
+        entity.cw_set(description=u'blurp')
+        cnx.commit()
+        result = ckan_post(self.ckan_config, 'package_show',
+                           {'id': entity.ckan_dataset_id})
+        resources = result['resources']
+        self.assertEqual(len(resources), 1)
 
     def _check_entity_delete(self, cnx, entity):
         self.set_description('entity deletion')