Load to ogp

README.rst

@@ -153,4 +153,17 @@ This field has two choices for dropdown list.
 
     The attribute 'form_restrict_choices_to' is for admin of organization and sysadmin only.
     The attribute 'choices' is for all members of organization.
-
+
+----------------------------------------------------------------------
+Add two flags to control ckanapi load to IDDP and ckanapi dump for OGP
+----------------------------------------------------------------------
+
+These two flags are set in ini config file.
+
+    ckan.ab_scheming.for_load_to_iddp = false
+    ckan.ab_scheming.for_dump_to_ogp = false
+
+    ckan.ab_scheming.for_load_to_iddp is for ckanapi load dataset of OGP to IDDP. If need to do so, set True. If not, set False.
+    ckan.ab_scheming.for_dump_to_ogp is for ckanapi dump dataset of IDDP for ckanapi load into OGP. If need to do so, set True. If not, set False.
+
+    These two flags are also controlled in run time in url /ckan-admin/config

ckanext/ab_scheming/logic/action/create.py

@@ -5,7 +5,7 @@
 
 
 def package_create(context, data_dict):
-    deployment_mode = toolkit.asbool(config.get('ckan.ab_scheming.deployment', False))
+    deployment_mode = toolkit.asbool(config.get('ckan.ab_scheming.for_load_to_iddp', False))
     # need to change data_dict if import from ckanapi
     if deployment_mode:
         data_dict = change_pkg_dict_for_import_deployment(data_dict, 'create')

ckanext/ab_scheming/logic/action/get.py

@@ -5,7 +5,8 @@
 import pylons.config as config
 import ckan.plugins.toolkit as toolkit
 import sqlalchemy
-
+from ckan.logic.action.get import package_show as _package_show
+import re
 
 _check_access = logic.check_access
 _select = sqlalchemy.sql.select
@@ -98,3 +99,153 @@ def topics_list_for_user(context, data_dict):
     for o in orgs_list:
         o['title'] = o['title'].decode('utf_8')
     return orgs_list
+
+
+@toolkit.side_effect_free
+def package_show(context, data_dict):
+    """
+    Cleanup package dict when 'ckanapi dump' is called for data transfer to OGP
+    """
+    package_dict = _package_show(context,data_dict)
+    # cleanup package dict
+    ogp_dump_mode = toolkit.asbool(config.get('ckan.ab_scheming.for_dump_to_ogp', False))
+    # need to change data_dict if import to OGA by ckanapi
+    if ogp_dump_mode:
+        package_dict = change_pkg_dict_for_dump_to_OGP(package_dict)
+    return package_dict
+
+
+def change_pkg_dict_for_dump_to_OGP(data_dict):
+    data_dict['url'] = ''
+
+    if 'topics' in data_dict:
+        if not data_dict['topics']: 
+            data_dict['topic'] = ['Environment']
+        else:
+            data_dict['topic'] = get_topic_name(data_dict['topics'])
+        del data_dict['topics']
+
+    if 'extras' in data_dict:
+        for e in data_dict['extras']:
+            if 'topics' == e['key']:
+                e['key'] = 'topic'
+                break
+
+    data_dict['id'] = ''
+
+    if 'groups' in data_dict:
+        data_dict['groups'] = []
+
+    if 'tags' in data_dict:
+        for ind, t in enumerate(data_dict['tags']):
+            data_dict['tags'][ind]['id'] = ''
+
+    if 'revision_id' in data_dict:
+        data_dict['revision_id'] = ''
+
+    if not 'organization' in data_dict or not data_dict['organization']:
+        data_dict['organization'] = {'name': 'servicealberta'}
+    if 'organization' in data_dict:
+        if data_dict['organization'].has_key('name'):
+            data_dict['organization']['name'] = data_dict['organization']['name'].encode('utf-8')
+        if data_dict['organization'].has_key('id'):
+            data_dict['organization']['id'] = ''
+        if data_dict['organization'].has_key('revision_id'):
+            data_dict['organization']['revision_id'] = ''
+
+    if not 'owner_org' in data_dict:
+        data_dict['owner_org'] = ''
+    if 'owner_org' in data_dict:
+        try:
+            data_dict['organization']['name']
+        except KeyError:
+            raise NotFound('Could not find organization name in dataset {0}'.format(data_dict['name']))
+        else:
+            data_dict['owner_org'] = data_dict['organization']['name']
+
+    if 'resources' in data_dict:
+        local_url = config.get('ckan.site_url', None)
+        for r in data_dict['resources']:
+            if 'classification' in r and int(r['classification']) > 1:
+                continue
+            if r.has_key('id'):
+                r['id'] = ''
+            if r.has_key('package_id'):
+                r['package_id'] = ''
+            if r.has_key('tracking_summary'):
+                del(r['tracking_summary'])
+            if r.has_key('classification'):
+                del(r['classification'])
+            if r.has_key('revision_id'):
+                r['revision_id'] = ''
+            if r.has_key('url_type') and \
+               re.search(r'{0}/dataset/.*?/resource/.*?/download/.*'.format(local_url), r['url']):
+                r['url_type'] = 'upload'
+            if r.has_key('webstore_last_updated'):
+                r['webstore_last_updated'] = ''
+            if r.has_key('webstore_url'):
+                r['webstore_url'] = ''
+
+    if 'process_state' in data_dict:
+        del(data_dict['private'])
+    if 'reason' in data_dict:
+        del(data_dict['reason'])
+
+    data_dict['creator_user_id'] = ''
+
+    if not 'creator' in data_dict or not data_dict['creator']:
+            data_dict['creator'] = ['Alberta Energy Regulator']
+    else:
+        tmp =[]
+        for c in data_dict['creator']:
+            c = c.encode('utf-8')
+            tmp.append(c)
+        data_dict['creator'] = tmp
+
+    if not 'contact_email' in data_dict or not data_dict['contact_email']:
+        data_dict['contact_email'] = 'NA'
+    if 'contact_email' in data_dict and data_dict['contact_email']:
+        data_dict['email'] = data_dict['contact_email']
+
+    if not 'contact' in data_dict or not data_dict['contact']:
+        data_dict['contact'] = 'NA'
+
+    if not 'updatefrequency' in data_dict or not data_dict['updatefrequency']:
+        data_dict['updatefrequency'] = 'Once'
+
+    if not 'sensitivity' in data_dict or not data_dict['sensitivity']:
+        data_dict['sensitivity'] = 'unrestricted'
+
+    if 'spatialcoverages' in data_dict:
+        del(data_dict['spatialcoverages'])
+
+    if 'last_process_state' in data_dict:
+        del(data_dict['last_process_state'])
+
+    if 'date_created' in data_dict and data_dict['type'] == 'publications':
+        data_dict['createdate'] = data_dict['date_created']
+        del(data_dict['date_created'])
+
+    return data_dict
+
+
+def get_topic_name(titles):
+    if len(titles) == 0:
+        return []
+    if not isinstance(titles, list):
+        titles = [titles]
+    tmp = []
+    for t in titles:
+        t = t.encode('utf-8')
+        try:
+            t_dict = toolkit.get_action('group_show')(data_dict = {
+                                                        'id': t,
+                                                        'type': 'topics'
+                                                    })
+        except NotFound:
+            raise NotFound("Topics {0} not found!".format(t))   
+
+        t = t_dict['title']
+        tmp.append(t)
+    return tmp
+

ckanext/ab_scheming/logic/action/update.py

@@ -5,7 +5,7 @@
 
 
 def package_update(context, data_dict):
-    deployment_mode = toolkit.asbool(config.get('ckan.ab_scheming.deployment', False))
+    deployment_mode = toolkit.asbool(config.get('ckan.ab_scheming.for_load_to_iddp', False))
     # need to change data_dict if import from ckanapi
     if deployment_mode:
         data_dict = change_pkg_dict_for_import_deployment(data_dict, 'update')

ckanext/ab_scheming/plugin.py

@@ -9,7 +9,8 @@
 )
 from ckan.logic.action  import get as ckan_get
 from ckanext.ab_scheming.logic.action.get import (
-    topics_list_for_user
+    topics_list_for_user,
+    package_show
 )
 from ckanext.ab_scheming.logic.action.create import package_create
 from ckanext.ab_scheming.logic.action.update import package_update
@@ -32,7 +33,21 @@ def update_config(self, config_):
         toolkit.add_template_directory(config_, 'templates')
         toolkit.add_public_directory(config_, 'public')
         toolkit.add_resource('fanstatic', 'ab_scheming')
-
+
+    # IConfigurer
+    def update_config_schema(self, schema):
+
+        ignore_missing = toolkit.get_validator('ignore_missing')
+        is_boolean = toolkit.get_validator('boolean_validator')
+
+        schema.update({
+            'ckan.ab_scheming.for_load_to_iddp': [ignore_missing, is_boolean],
+            'ckan.ab_scheming.for_dump_to_ogp': [ignore_missing, is_boolean],
+        })
+
+        return schema
+
+
     def dataset_facets(self, facets_dict, package_type):
         facets_dict['dataset_type'] = plugins.toolkit._('Information Type')
         facets_dict['groups'] = plugins.toolkit._('Topics')
@@ -71,6 +86,7 @@ def get_actions(self):
                        if callable(function))
         '''
         actions = {'topics_list_for_user': topics_list_for_user,
+                    'package_show': package_show,
                     'package_create': package_create,
                     'package_update': package_update
                   }

ckanext/ab_scheming/templates/admin/config.html

@@ -0,0 +1,27 @@
+{% ckan_extends %}
+
+{% import 'macros/form.html' as form %}
+
+{% block admin_form %}
+
+  {{ super() }}
+
+  <h3>Custom configuration options </h3>
+
+  {{ form.input('ckan.ab_scheming.for_load_to_iddp', id='field-ckan.ab_scheming.for_load_to_iddp', label=_('Load to IDDP with OGP dataset'), value=data['ckan.ab_scheming.for_load_to_iddp'], error=errors['ckan.ab_scheming.for_load_to_iddp']) }}
+
+  {{ form.input('ckan.ab_scheming.for_dump_to_ogp', id='field-ckan.ab_scheming.for_dump_to_ogp', label=_('Dump dataset of IDDP to OGP'), value=data['ckan.ab_scheming.for_dump_to_ogp'], error=errors['ckan.ab_scheming.for_dump_to_ogp']) }}
+
+
+{% endblock %}
+
+
+{% block admin_form_help %}
+
+  {{ super() }}
+
+  <p><strong>Load to IDDP with OGP dataset:</strong> Set to True when needs to ckanapi load Open Government Portal's datasets to IDDP. After that, needs to set it False when load is done to make the server function properly.</p>
+
+  <p><strong>Dump dataset of IDDP to OGP:</strong> Set to True when needs to ckanapi dump IDDP datasets for use of ckanapi load to OGP. Set it False when dump is done to make the server function properly.</p>
+
+{% endblock %}