pycsw

geonode/catalogue/__init__.py

@@ -18,7 +18,7 @@
 #########################################################################
 
 """
-Tools for managing a CatalogWebService (CSW)
+Tools for managing a Catalogue Service for the Web (CSW)
 """
 import os
 from django.conf import settings

geonode/catalogue/backends/generic.py

@@ -29,20 +29,21 @@
 from geonode.catalogue.backends.base import BaseCatalogueBackend
 
 METADATA_FORMATS = {
+    'Atom': ('atom:entry', 'http://www.w3.org/2005/Atom'),
     'DIF': ('dif:DIF', 'http://gcmd.gsfc.nasa.gov/Aboutus/xml/dif/'),
     'Dublin Core': ('csw:Record', 'http://www.opengis.net/cat/csw/2.0.2'),
+    'ebRIM': ('rim:RegistryObject', 'urn:oasis:names:tc:ebxml-regrep:xsd:rim:3.0'),
     'FGDC': ('fgdc:metadata', 'http://www.opengis.net/cat/csw/csdgm'),
     'TC211': ('gmd:MD_Metadata', 'http://www.isotc211.org/2005/gmd'),
 }
 
-
-
 class Catalogue(CatalogueServiceWeb):
     def __init__(self, *args, **kwargs):
         self.url = kwargs['URL']
         self.user = kwargs['USER']
         self.password = kwargs['PASSWORD']
-        self.formats = kwargs['FORMATS']
+        self.type = kwargs['ENGINE'].split('.')[-1]
+        self.local = False
         self._group_ids = {}
         self._operation_ids = {}
         self.connected = False
@@ -122,8 +123,7 @@ def urls_for_uuid(self, uuid):
             urls.append(('text/xml', mformat, self.url_for_uuid(uuid, METADATA_FORMATS[mformat][1])))
         return urls
 
-    def csw_request(self, layer, template):
-
+    def csw_gen_xml(self, layer, template):
         id_pname = 'dc:identifier'
         if self.type == 'deegree':
             id_pname = 'apiso:Identifier'
@@ -136,6 +136,16 @@ def csw_request(self, layer, template):
         })
         md_doc = tpl.render(ctx)
         md_doc = md_doc.encode("utf-8")
+        return md_doc
+
+    def csw_gen_anytext(self, xml):
+        """ get all element data from an XML document """
+        xml = etree.fromstring(xml)
+        return ' '.join([value for value in xml.xpath('//text()')])
+
+    def csw_request(self, layer, template):
+
+        md_doc = self.csw_gen_xml(layer, template)
 
         if self.type == 'geonetwork':
             headers = {
@@ -356,8 +366,6 @@ def extract_links(self, rec):
                     pass
         return links
 
-
-
 class CatalogueBackend(BaseCatalogueBackend):
     def __init__(self, *args, **kwargs):
        self.catalogue = Catalogue(*args, **kwargs) 
@@ -371,7 +379,6 @@ def get_record(self, uuid):
                 rec.links['download'] = self.catalogue.extract_links(rec)
         return rec
 
-
     def search_records(self, keywords, start, limit, bbox):
         with self.catalogue: 
             bbox = self.catalogue.normalize_bbox(bbox)
@@ -388,13 +395,12 @@ def search_records(self, keywords, start, limit, bbox):
 
             return result
 
-
     def remove_record(self, uuid):
         with self.catalogue:
            catalogue_record = self.catalogue.get_by_uuid(uuid)
            if catalogue_record is None:
                return
-
+    
            try:
                # this is a bit hacky, delete_layer expects an instance of the layer
                # model but it just passes it to a Django template so a dict works

geonode/catalogue/backends/geonetwork.py

@@ -22,4 +22,4 @@
 class CatalogueBackend(GenericCatalogueBackend):
     def __init__(self, *args, **kwargs):
         super(CatalogueBackend, self).__init__(*args, **kwargs)
-        self.catalogue.type = 'geonetwork'
+        self.catalogue.formats = ['Dublin Core', 'TC211']

geonode/catalogue/backends/pycsw_http.py

@@ -0,0 +1,26 @@
+#########################################################################
+#
+# Copyright (C) 2012 OpenPlans
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+#########################################################################
+
+from django.conf import settings
+from geonode.catalogue.backends.generic import CatalogueBackend as GenericCatalogueBackend
+
+class CatalogueBackend(GenericCatalogueBackend):
+    def __init__(self, *args, **kwargs):
+        super(CatalogueBackend, self).__init__(*args, **kwargs)
+        self.catalogue.formats = ['Atom', 'DIF', 'Dublin Core', 'ebRIM', 'FGDC', 'TC211']

geonode/catalogue/backends/pycsw_local.py

@@ -0,0 +1,190 @@
+#########################################################################
+#
+# Copyright (C) 2012 OpenPlans
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+#########################################################################
+
+import os
+from lxml import etree
+from django.conf import settings
+from ConfigParser import SafeConfigParser
+from owslib.iso import MD_Metadata
+from pycsw import server
+from geonode.catalogue.backends.generic import CatalogueBackend as GenericCatalogueBackend
+from geonode.catalogue.backends.generic import METADATA_FORMATS
+
+MD_CORE_MODEL = {
+    'typename': 'pycsw:CoreMetadata',
+    'outputschema': 'http://pycsw.org/metadata',
+    'mappings': {
+        'pycsw:Identifier': 'uuid',
+        'pycsw:Typename': 'csw_typename',
+        'pycsw:Schema': 'csw_schema',
+        'pycsw:MdSource': 'csw_mdsource',
+        'pycsw:InsertDate': 'csw_insert_date',
+        'pycsw:XML': 'metadata_xml',
+        'pycsw:AnyText': 'csw_anytext',
+        'pycsw:Language': 'language',
+        'pycsw:Title': 'title',
+        'pycsw:Abstract': 'abstract',
+        'pycsw:Keywords': 'keyword_csv',
+        'pycsw:KeywordType': 'keywordstype',
+        'pycsw:Format': 'spatial_representation_type',
+        'pycsw:Source': 'source',
+        'pycsw:Date': 'date',
+        'pycsw:Modified': 'last_modified',
+        'pycsw:Type': 'csw_type',
+        'pycsw:BoundingBox': 'csw_wkt_geometry',
+        'pycsw:CRS': 'crs',
+        'pycsw:AlternateTitle': 'title_alternate',
+        'pycsw:RevisionDate': 'date_revision',
+        'pycsw:CreationDate': 'last_modified',
+        'pycsw:PublicationDate': 'date_publication',
+        'pycsw:OrganizationName': 'uuid',
+        'pycsw:SecurityConstraints': 'securityconstraints',
+        'pycsw:ParentIdentifier': 'parentidentifier',
+        'pycsw:TopicCategory': 'topicategory',
+        'pycsw:ResourceLanguage': 'resourcelanguage',
+        'pycsw:GeographicDescriptionCode': 'geodescode',
+        'pycsw:Denominator': 'denominator',
+        'pycsw:DistanceValue': 'distancevalue',
+        'pycsw:DistanceUOM': 'distanceuom',
+        'pycsw:TempExtent_begin': 'temporal_extent_start',
+        'pycsw:TempExtent_end': 'temporal_extent_end',
+        'pycsw:ServiceType': 'servicetype',
+        'pycsw:ServiceTypeVersion': 'servicetypeversion',
+        'pycsw:Operation': 'operation',
+        'pycsw:CouplingType': 'couplingtype',
+        'pycsw:OperatesOn': 'operateson',
+        'pycsw:OperatesOnIdentifier': 'operatesonidentifier',
+        'pycsw:OperatesOnName': 'operatesoname',
+        'pycsw:Degree': 'degree',
+        'pycsw:AccessConstraints': 'accessconstraints',
+        'pycsw:OtherConstraints': 'otherconstraints',
+        'pycsw:Classification': 'classification',
+        'pycsw:ConditionApplyingToAccessAndUse': 'conditionapplyingtoaccessanduse',
+        'pycsw:Lineage': 'lineage',
+        'pycsw:ResponsiblePartyRole': 'responsiblepartyrole',
+        'pycsw:SpecificationTitle': 'specificationtitle',
+        'pycsw:SpecificationDate': 'specificationdate',
+        'pycsw:SpecificationDateType': 'specificationdatetype',
+        'pycsw:Creator': 'creator',
+        'pycsw:Publisher': 'publisher',
+        'pycsw:Contributor': 'contributor',
+        'pycsw:Relation': 'relation',
+        'pycsw:Links': 'download_links',
+    }
+}
+
+class CatalogueBackend(GenericCatalogueBackend):
+    def __init__(self, *args, **kwargs):
+        super(CatalogueBackend, self).__init__(*args, **kwargs)
+        self.catalogue.formats = ['Atom', 'DIF', 'Dublin Core', 'ebRIM', 'FGDC', 'TC211']
+        self.catalogue.local = True
+
+    def remove_record(self, uuid):
+        pass
+
+    def create_record(self, item):
+        pass
+
+    def get_record(self, uuid):
+        results = self._csw_local_dispatch(identifier=uuid)
+        if len(results) < 1:
+            return None
+
+        result = etree.fromstring(results).find('{http://www.isotc211.org/2005/gmd}MD_Metadata')
+
+        if result is None:
+            return None
+
+        record = MD_Metadata(result)
+        record.keywords = []
+        if hasattr(record, 'identification') and hasattr(record.identification, 'keywords'):
+            for kw in record.identification.keywords:
+                record.keywords.extend(kw['keywords'])
+
+        record.links = {}
+        record.links['metadata'] = self.catalogue.urls_for_uuid(uuid)
+        record.links['download'] = self.catalogue.extract_links(record)
+        return record
+
+    def search_records(self, keywords, start, limit, bbox):
+        with self.catalogue:
+            lresults = self._csw_local_dispatch(keywords, keywords, start+1, limit, bbox)
+            # serialize XML
+            e = etree.fromstring(lresults)
+            self.catalogue.records = [MD_Metadata(x) for x in e.findall('//{http://www.isotc211.org/2005/gmd}MD_Metadata')]
+
+            # build results into JSON for API
+            results = [self.catalogue.metadatarecord2dict(doc) for v, doc in self.catalogue.records.iteritems()]
+
+            result = {
+                      'rows': results,
+                      'total': e.find('{http://www.opengis.net/cat/csw/2.0.2}SearchResults').attrib.get('numberOfRecordsMatched'),
+                      'next_page': e.find('{http://www.opengis.net/cat/csw/2.0.2}SearchResults').attrib.get('nextRecord')
+                      }
+
+            return result
+
+
+    def _csw_local_dispatch(self, keywords=None, start=0, limit=10, bbox=None, identifier=None):
+        """
+        HTTP-less CSW
+        """
+        # set up configuration
+        config = SafeConfigParser()
+
+        for section, options in settings.PYCSW['CONFIGURATION'].iteritems():
+            config.add_section(section)
+            for option, value in options.iteritems():
+                config.set(section, option, value)
+
+        # fake HTTP environment variable
+        os.environ['QUERY_STRING'] = ''
+
+        # init pycsw
+        csw = server.Csw(config)
+
+        # fake HTTP method
+        csw.requesttype = 'POST'
+
+        # fake HTTP request parameters
+        if identifier is None:  # it's a GetRecords request
+            formats = []
+            for f in self.catalogue.formats:
+                formats.append(METADATA_FORMATS[f][0])
+
+            csw.kvp = {
+                'elementsetname': 'full',
+                'typenames': formats,
+                'resulttype': 'results',
+                'constraintlanguage': 'CQL_TEXT',
+                'constraint': 'csw:AnyText like "%%%s%%"' % keywords,
+                'outputschema': 'http://www.isotc211.org/2005/gmd',
+                'constraint': None,
+                'startposition': start,
+                'maxrecords': limit
+            }
+            response = csw.getrecords()
+        else:  # it's a GetRecordById request
+            csw.kvp = {
+                'id': [identifier],
+                'outputschema': 'http://www.isotc211.org/2005/gmd',
+            }
+            response = csw.getrecordbyid()
+
+        return etree.tostring(response)

geonode/catalogue/models.py

@@ -64,7 +64,18 @@ def catalogue_post_save(instance, sender, **kwargs):
                           )
                          )
 
+    # generate and save CSW specific fields
+    signals.post_save.disconnect(catalogue_post_save, sender=Layer)
 
+    md_doc = catalogue.catalogue.csw_gen_xml(instance, 'catalogue/full_metadata.xml')
+    instance.metadata_xml = md_doc
+    instance.csw_anytext = catalogue.catalogue.csw_gen_anytext(md_doc)
+
+    instance.csw_wkt_geometry = instance.geographic_bounding_box
+
+    instance.save()
+
+    signals.post_save.connect(catalogue_post_save, sender=Layer)
 
 def catalogue_pre_save(instance, sender, **kwargs):
     """Send information to catalogue

geonode/catalogue/urls.py

@@ -0,0 +1,25 @@
+# -*- coding: utf-8 -*-
+#########################################################################
+#
+# Copyright (C) 2012 OpenPlans
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+#########################################################################
+
+from django.conf.urls.defaults import patterns, url
+
+urlpatterns = patterns('geonode.catalogue.views',
+    url(r'^csw$', 'csw_global_dispatch', name='csw_global_dispatch'),
+)