From 9e95a01d5ab70ddf647ac81e70fd16a4b11b830c Mon Sep 17 00:00:00 2001 From: Ian Schneider Date: Wed, 17 Oct 2012 16:17:17 -0600 Subject: [PATCH 1/2] silage - db driven search backend --- geonode/search/__init__.py | 0 geonode/search/backends/__init__.py | 0 geonode/search/backends/silage/__init__.py | 2 + geonode/search/backends/silage/extension.py | 66 ++ .../silage/fixtures/silage_testdata.json | 1036 +++++++++++++++++ geonode/search/backends/silage/geomodels.py | 221 ++++ .../backends/silage/management/__init__.py | 0 .../silage/management/commands/__init__.py | 0 .../update_spatial_temporal_indices.py | 52 + geonode/search/backends/silage/models.py | 60 + geonode/search/backends/silage/normalizers.py | 202 ++++ .../silage/populate_search_test_data.py | 129 ++ geonode/search/backends/silage/query.py | 197 ++++ geonode/search/backends/silage/search.py | 288 +++++ geonode/search/backends/silage/tests.py | 305 +++++ geonode/search/backends/silage/urls.py | 28 + geonode/search/backends/silage/util.py | 185 +++ geonode/search/backends/silage/views.py | 227 ++++ 18 files changed, 2998 insertions(+) create mode 100644 geonode/search/__init__.py create mode 100644 geonode/search/backends/__init__.py create mode 100644 geonode/search/backends/silage/__init__.py create mode 100644 geonode/search/backends/silage/extension.py create mode 100644 geonode/search/backends/silage/fixtures/silage_testdata.json create mode 100644 geonode/search/backends/silage/geomodels.py create mode 100644 geonode/search/backends/silage/management/__init__.py create mode 100644 geonode/search/backends/silage/management/commands/__init__.py create mode 100644 geonode/search/backends/silage/management/commands/update_spatial_temporal_indices.py create mode 100644 geonode/search/backends/silage/models.py create mode 100644 geonode/search/backends/silage/normalizers.py create mode 100644 geonode/search/backends/silage/populate_search_test_data.py create mode 100644 geonode/search/backends/silage/query.py create mode 100644 geonode/search/backends/silage/search.py create mode 100644 geonode/search/backends/silage/tests.py create mode 100644 geonode/search/backends/silage/urls.py create mode 100644 geonode/search/backends/silage/util.py create mode 100644 geonode/search/backends/silage/views.py diff --git a/geonode/search/__init__.py b/geonode/search/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/search/backends/__init__.py b/geonode/search/backends/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/search/backends/silage/__init__.py b/geonode/search/backends/silage/__init__.py new file mode 100644 index 00000000000..139597f9cb0 --- /dev/null +++ b/geonode/search/backends/silage/__init__.py @@ -0,0 +1,2 @@ + + diff --git a/geonode/search/backends/silage/extension.py b/geonode/search/backends/silage/extension.py new file mode 100644 index 00000000000..461b5d4f3ed --- /dev/null +++ b/geonode/search/backends/silage/extension.py @@ -0,0 +1,66 @@ +######################################################################### +# +# Copyright (C) 2012 OpenPlans +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from geonode.people.models import Contact +from geonode.layers.models import Layer +from geonode.maps.models import Map +from geonode.search.backends.silage.util import resolve_extension + +from django.conf import settings +import re + +date_fmt = lambda dt: dt.isoformat() +USER_DISPLAY = 'User' +MAP_DISPLAY = 'Map' +LAYER_DISPLAY = 'Layer' + +# settings API +search_config = getattr(settings,'SIMPLE_SEARCH_SETTINGS', {}) + +exclude_patterns = search_config.get('layer_exclusions',[]) + +exclude_regex = [ re.compile(e) for e in exclude_patterns ] + +process_results = resolve_extension('process_search_results') +if process_results is None: + process_results = lambda r: r + +owner_query = resolve_extension('owner_query') +if not owner_query: + owner_query = lambda q: Contact.objects.filter() + +owner_query_fields = resolve_extension('owner_query_fields') or [] + +layer_query = resolve_extension('layer_query') +if not layer_query: + layer_query = lambda q: Layer.objects.filter() + +map_query = resolve_extension('map_query') +if not map_query: + map_query = lambda q: Map.objects.filter() + +display_names = resolve_extension('display_names') +if display_names: + USER_DISPLAY = display_names.get('user') + MAP_DISPLAY = display_names.get('map') + LAYER_DISPLAY = display_names.get('layer') + +owner_rank_rules = resolve_extension('owner_rank_rules') +if not owner_rank_rules: + owner_rank_rules = lambda: [] \ No newline at end of file diff --git a/geonode/search/backends/silage/fixtures/silage_testdata.json b/geonode/search/backends/silage/fixtures/silage_testdata.json new file mode 100644 index 00000000000..68c359798b8 --- /dev/null +++ b/geonode/search/backends/silage/fixtures/silage_testdata.json @@ -0,0 +1,1036 @@ +[ + { + "pk": 1, + "model": "auth.user", + "fields": { + "username": "admin", + "first_name": "", + "last_name": "", + "is_active": true, + "is_superuser": true, + "is_staff": true, + "last_login": "2012-09-28T17:26:42.320", + "groups": [], + "user_permissions": [], + "password": "pbkdf2_sha256$10000$LaaGZBGeJ7Ew$XjMtWwwDMcolUJwO7mabwLvYIkbCYpo0hiFCUvK35nY=", + "email": "a@d.min", + "date_joined": "2012-09-28T17:26:42.320" + } + }, + { + "pk": 2, + "model": "auth.user", + "fields": { + "username": "user1", + "first_name": "uniquefirst", + "last_name": "foo", + "is_active": true, + "is_superuser": false, + "is_staff": false, + "last_login": "2012-09-28T19:19:32.632", + "groups": [], + "user_permissions": [], + "password": "!", + "email": "", + "date_joined": "2012-09-28T19:19:32.632" + } + }, + { + "pk": 3, + "model": "auth.user", + "fields": { + "username": "user2", + "first_name": "foo", + "last_name": "uniquelast", + "is_active": true, + "is_superuser": false, + "is_staff": false, + "last_login": "2012-09-28T19:19:33.296", + "groups": [], + "user_permissions": [], + "password": "!", + "email": "", + "date_joined": "2012-09-28T19:19:33.296" + } + }, + { + "pk": 4, + "model": "auth.user", + "fields": { + "username": "unique_username", + "first_name": "foo", + "last_name": "uniquelast", + "is_active": true, + "is_superuser": false, + "is_staff": false, + "last_login": "2012-09-28T19:19:33.843", + "groups": [], + "user_permissions": [], + "password": "!", + "email": "", + "date_joined": "2012-09-28T19:19:33.843" + } + }, + { + "pk": 5, + "model": "auth.user", + "fields": { + "username": "jblaze", + "first_name": "johnny", + "last_name": "blaze", + "is_active": true, + "is_superuser": false, + "is_staff": false, + "last_login": "2012-09-28T19:19:34.376", + "groups": [], + "user_permissions": [], + "password": "!", + "email": "", + "date_joined": "2012-09-28T19:19:34.376" + } + }, + { + "pk": 6, + "model": "auth.user", + "fields": { + "username": "foo", + "first_name": "bar", + "last_name": "baz", + "is_active": true, + "is_superuser": false, + "is_staff": false, + "last_login": "2012-09-28T19:19:34.964", + "groups": [], + "user_permissions": [], + "password": "!", + "email": "", + "date_joined": "2012-09-28T19:19:34.964" + } + }, + { + "pk": 1, + "model": "people.contact", + "fields": { + "profile": null, + "city": null, + "fax": null, + "name": "admin", + "area": null, + "country": null, + "zipcode": null, + "delivery": null, + "user": [ + "admin" + ], + "position": null, + "organization": null, + "voice": null, + "email": null + } + }, + { + "pk": 2, + "model": "people.contact", + "fields": { + "profile": "this contains all my interesting profile information", + "city": null, + "fax": null, + "name": "user1", + "area": null, + "country": null, + "zipcode": null, + "delivery": null, + "user": [ + "user1" + ], + "position": null, + "organization": null, + "voice": null, + "email": null + } + }, + { + "pk": 3, + "model": "people.contact", + "fields": { + "profile": "some other information goes here", + "city": null, + "fax": null, + "name": "user2", + "area": null, + "country": null, + "zipcode": null, + "delivery": null, + "user": [ + "user2" + ], + "position": null, + "organization": null, + "voice": null, + "email": null + } + }, + { + "pk": 4, + "model": "people.contact", + "fields": { + "profile": "this contains all my interesting profile information", + "city": null, + "fax": null, + "name": "unique_username", + "area": null, + "country": null, + "zipcode": null, + "delivery": null, + "user": [ + "unique_username" + ], + "position": null, + "organization": null, + "voice": null, + "email": null + } + }, + { + "pk": 5, + "model": "people.contact", + "fields": { + "profile": "some other information goes here", + "city": null, + "fax": null, + "name": "jblaze", + "area": null, + "country": null, + "zipcode": null, + "delivery": null, + "user": [ + "jblaze" + ], + "position": null, + "organization": null, + "voice": null, + "email": null + } + }, + { + "pk": 6, + "model": "people.contact", + "fields": { + "profile": "this contains all my interesting profile information", + "city": null, + "fax": null, + "name": "foo", + "area": null, + "country": null, + "zipcode": null, + "delivery": null, + "user": [ + "foo" + ], + "position": null, + "organization": null, + "voice": null, + "email": null + } + }, + { + "pk": 1, + "model": "layers.layer", + "fields": { + "constraints_other": null, + "bbox_x0": "-180", + "csw_mdsource": "local", + "bbox_y1": "90", + "csw_typename": "gmd:MD_Metadata", + "date_type": "publication", + "srid": "EPSG:4326", + "bbox_x1": "180", + "metadata_xml": "\n \n 95632946-3f41-4dae-9c85-35c7f8fd7b50\n \n \n eng\n \n \n utf8\n \n \n dataset\n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/\n \n \n WWW:LINK-1.0-http--link\n \n \n GeoNode profile page\n \n \n \n \n \n \n \n pointOfContact\n \n \n \n \n 2012-09-28T19:19:40Z\n \n \n ISO 19115:2003 - Geographic information - Metadata\n \n \n ISO 19115:2003\n \n \n \n \n \n \n \n 4326\n \n \n EPSG\n \n \n 6.11\n \n \n \n \n \n \n \n \n \n \n layer1\n \n \n \n \n 2012-09-28T19:19:40Z\n \n \n publication\n \n \n \n \n \n \n \n mapDigital\n \n \n \n \n abstract1\n \n \n \n \n \n completed\n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Alayer1&width=20&version=1.1.1&bbox=-180%2C180%2C-90%2C90&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20\n \n \n Thumbnail for 'layer1'\n \n \n image/png\n \n \n \n \n \n \n \n ESRI Shapefile\n \n \n \n 1.0\n \n \n \n \n \n \n \n \n populartag\n \n \n \n here\n \n \n \n theme\n \n \n \n \n \n \n \n \n United States of America\n \n \n place\n \n \n \n \n \n \n \n copyright\n \n \n \n \n None\n \n \n eng\n \n \n utf8\n \n \n \n location\n \n \n \n \n \n \n \n -180\n \n \n 180\n \n \n -90\n \n \n 90\n \n \n \n \n \n \n \n \n \n \n \n \n Jan. 1, 1985, midnight\n Jan. 1, 1986, midnight\n \n \n \n \n \n \n \n \n No information provided\n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:layer1\n \n \n WWW:LINK-1.0-http--link\n \n \n Online link to the 'layer1' description on GeoNode\n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset\n \n \n \n \n \n \n \n \n \n \n \n", + "edition": null, + "owner": [ + "user1" + ], + "distribution_url": "http://localhost:8000/data/geonode:layer1", + "spatial_representation_type": null, + "uuid": "95632946-3f41-4dae-9c85-35c7f8fd7b50", + "title": "layer1", + "csw_schema": "http://www.isotc211.org/2005/gmd", + "storeType": "raster", + "abstract": "abstract1", + "store": "", + "bbox_y0": "-90", + "distribution_description": "layer1", + "topic_category": "location", + "csw_insert_date": "2012-09-28T19:19:40.555", + "purpose": null, + "date": "2012-09-28T19:19:40.176", + "name": "layer1", + "temporal_extent_end": "1986-01-01", + "data_quality_statement": null, + "language": "eng", + "keywords_region": "USA", + "maintenance_frequency": null, + "csw_anytext": "\n \n 95632946-3f41-4dae-9c85-35c7f8fd7b50 \n \n \n eng \n \n \n utf8 \n \n \n dataset \n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/ \n \n \n WWW:LINK-1.0-http--link \n \n \n GeoNode profile page \n \n \n \n \n \n \n \n pointOfContact \n \n \n \n \n 2012-09-28T19:19:40Z \n \n \n ISO 19115:2003 - Geographic information - Metadata \n \n \n ISO 19115:2003 \n \n \n \n \n \n \n \n 4326 \n \n \n EPSG \n \n \n 6.11 \n \n \n \n \n \n \n \n \n \n \n layer1 \n \n \n \n \n 2012-09-28T19:19:40Z \n \n \n publication \n \n \n \n \n \n \n \n mapDigital \n \n \n \n \n abstract1 \n \n \n \n \n \n completed \n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Alayer1&width=20&version=1.1.1&bbox=-180%2C180%2C-90%2C90&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20 \n \n \n Thumbnail for 'layer1' \n \n \n image/png \n \n \n \n \n \n \n \n ESRI Shapefile \n \n \n \n 1.0 \n \n \n \n \n \n \n \n \n populartag \n \n \n \n here \n \n \n \n theme \n \n \n \n \n \n \n \n \n United States of America \n \n \n place \n \n \n \n \n \n \n \n copyright \n \n \n \n \n None \n \n \n eng \n \n \n utf8 \n \n \n \n location \n \n \n \n \n \n \n \n -180 \n \n \n 180 \n \n \n -90 \n \n \n 90 \n \n \n \n \n \n \n \n \n \n \n \n \n Jan. 1, 1985, midnight \n Jan. 1, 1986, midnight \n \n \n \n \n \n \n \n \n No information provided \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:layer1 \n \n \n WWW:LINK-1.0-http--link \n \n \n Online link to the 'layer1' description on GeoNode \n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset \n \n \n \n \n \n \n \n \n \n \n ", + "supplemental_information": "No information provided", + "typename": "geonode:layer1", + "csw_type": "dataset", + "workspace": "", + "metadata_uploaded": false, + "csw_wkt_geometry": "SRID=EPSG:4326;POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))", + "temporal_extent_start": "1985-01-01", + "constraints_use": "copyright" + } + }, + { + "pk": 2, + "model": "layers.layer", + "fields": { + "constraints_other": null, + "bbox_x0": "-180", + "csw_mdsource": "local", + "bbox_y1": "90", + "csw_typename": "gmd:MD_Metadata", + "date_type": "publication", + "srid": "EPSG:4326", + "bbox_x1": "180", + "metadata_xml": "\n \n 747926c4-cc41-4a6f-8127-57528571f0ee\n \n \n eng\n \n \n utf8\n \n \n dataset\n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/\n \n \n WWW:LINK-1.0-http--link\n \n \n GeoNode profile page\n \n \n \n \n \n \n \n pointOfContact\n \n \n \n \n 2012-09-28T19:19:43Z\n \n \n ISO 19115:2003 - Geographic information - Metadata\n \n \n ISO 19115:2003\n \n \n \n \n \n \n \n 4326\n \n \n EPSG\n \n \n 6.11\n \n \n \n \n \n \n \n \n \n \n layer2\n \n \n \n \n 2012-09-28T19:19:43Z\n \n \n publication\n \n \n \n \n \n \n \n mapDigital\n \n \n \n \n abstract2\n \n \n \n \n \n completed\n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Alayer2&width=20&version=1.1.1&bbox=-180%2C180%2C-90%2C90&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20\n \n \n Thumbnail for 'layer2'\n \n \n image/png\n \n \n \n \n \n \n \n ESRI Shapefile\n \n \n \n 1.0\n \n \n \n \n \n \n \n \n populartag\n \n \n \n theme\n \n \n \n \n \n \n \n \n United States of America\n \n \n place\n \n \n \n \n \n \n \n copyright\n \n \n \n \n None\n \n \n eng\n \n \n utf8\n \n \n \n location\n \n \n \n \n \n \n \n -180\n \n \n 180\n \n \n -90\n \n \n 90\n \n \n \n \n \n \n \n \n \n \n \n \n May 1, 1980, midnight\n May 1, 1981, midnight\n \n \n \n \n \n \n \n \n No information provided\n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:layer2\n \n \n WWW:LINK-1.0-http--link\n \n \n Online link to the 'layer2' description on GeoNode\n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset\n \n \n \n \n \n \n \n \n \n \n \n", + "edition": null, + "owner": [ + "user2" + ], + "distribution_url": "http://localhost:8000/data/geonode:layer2", + "spatial_representation_type": null, + "uuid": "747926c4-cc41-4a6f-8127-57528571f0ee", + "title": "layer2", + "csw_schema": "http://www.isotc211.org/2005/gmd", + "storeType": "vector", + "abstract": "abstract2", + "store": "", + "bbox_y0": "-90", + "distribution_description": "layer2", + "topic_category": "location", + "csw_insert_date": "2012-09-28T19:19:43.525", + "purpose": null, + "date": "2012-09-28T19:19:43.237", + "name": "layer2", + "temporal_extent_end": "1981-05-01", + "data_quality_statement": null, + "language": "eng", + "keywords_region": "USA", + "maintenance_frequency": null, + "csw_anytext": "\n \n 747926c4-cc41-4a6f-8127-57528571f0ee \n \n \n eng \n \n \n utf8 \n \n \n dataset \n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/ \n \n \n WWW:LINK-1.0-http--link \n \n \n GeoNode profile page \n \n \n \n \n \n \n \n pointOfContact \n \n \n \n \n 2012-09-28T19:19:43Z \n \n \n ISO 19115:2003 - Geographic information - Metadata \n \n \n ISO 19115:2003 \n \n \n \n \n \n \n \n 4326 \n \n \n EPSG \n \n \n 6.11 \n \n \n \n \n \n \n \n \n \n \n layer2 \n \n \n \n \n 2012-09-28T19:19:43Z \n \n \n publication \n \n \n \n \n \n \n \n mapDigital \n \n \n \n \n abstract2 \n \n \n \n \n \n completed \n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Alayer2&width=20&version=1.1.1&bbox=-180%2C180%2C-90%2C90&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20 \n \n \n Thumbnail for 'layer2' \n \n \n image/png \n \n \n \n \n \n \n \n ESRI Shapefile \n \n \n \n 1.0 \n \n \n \n \n \n \n \n \n populartag \n \n \n \n theme \n \n \n \n \n \n \n \n \n United States of America \n \n \n place \n \n \n \n \n \n \n \n copyright \n \n \n \n \n None \n \n \n eng \n \n \n utf8 \n \n \n \n location \n \n \n \n \n \n \n \n -180 \n \n \n 180 \n \n \n -90 \n \n \n 90 \n \n \n \n \n \n \n \n \n \n \n \n \n May 1, 1980, midnight \n May 1, 1981, midnight \n \n \n \n \n \n \n \n \n No information provided \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:layer2 \n \n \n WWW:LINK-1.0-http--link \n \n \n Online link to the 'layer2' description on GeoNode \n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset \n \n \n \n \n \n \n \n \n \n \n ", + "supplemental_information": "No information provided", + "typename": "geonode:layer2", + "csw_type": "dataset", + "workspace": "", + "metadata_uploaded": false, + "csw_wkt_geometry": "SRID=EPSG:4326;POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))", + "temporal_extent_start": "1980-05-01", + "constraints_use": "copyright" + } + }, + { + "pk": 3, + "model": "layers.layer", + "fields": { + "constraints_other": null, + "bbox_x0": "-180", + "csw_mdsource": "local", + "bbox_y1": "90", + "csw_typename": "gmd:MD_Metadata", + "date_type": "publication", + "srid": "EPSG:4326", + "bbox_x1": "180", + "metadata_xml": "\n \n 7373d024-4273-4c6b-9a36-41878f4ce67b\n \n \n eng\n \n \n utf8\n \n \n dataset\n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/\n \n \n WWW:LINK-1.0-http--link\n \n \n GeoNode profile page\n \n \n \n \n \n \n \n pointOfContact\n \n \n \n \n 2012-09-28T19:19:45Z\n \n \n ISO 19115:2003 - Geographic information - Metadata\n \n \n ISO 19115:2003\n \n \n \n \n \n \n \n 4326\n \n \n EPSG\n \n \n 6.11\n \n \n \n \n \n \n \n \n \n \n uniquetitle\n \n \n \n \n 2012-09-28T19:19:45Z\n \n \n publication\n \n \n \n \n \n \n \n mapDigital\n \n \n \n \n something here\n \n \n \n \n \n completed\n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Amylayer&width=20&version=1.1.1&bbox=-180%2C180%2C-90%2C90&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20\n \n \n Thumbnail for 'uniquetitle'\n \n \n image/png\n \n \n \n \n \n \n \n ESRI Shapefile\n \n \n \n 1.0\n \n \n \n \n \n \n \n \n populartag\n \n \n \n theme\n \n \n \n \n \n \n \n \n United States of America\n \n \n place\n \n \n \n \n \n \n \n copyright\n \n \n \n \n None\n \n \n eng\n \n \n utf8\n \n \n \n location\n \n \n \n \n \n \n \n -180\n \n \n 180\n \n \n -90\n \n \n 90\n \n \n \n \n \n \n \n \n \n \n \n \n Oct. 1, 1990, midnight\n Oct. 1, 1991, midnight\n \n \n \n \n \n \n \n \n No information provided\n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:mylayer\n \n \n WWW:LINK-1.0-http--link\n \n \n Online link to the 'uniquetitle' description on GeoNode\n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset\n \n \n \n \n \n \n \n \n \n \n \n", + "edition": null, + "owner": [ + "unique_username" + ], + "distribution_url": "http://localhost:8000/data/geonode:mylayer", + "spatial_representation_type": null, + "uuid": "7373d024-4273-4c6b-9a36-41878f4ce67b", + "title": "uniquetitle", + "csw_schema": "http://www.isotc211.org/2005/gmd", + "storeType": "raster", + "abstract": "something here", + "store": "", + "bbox_y0": "-90", + "distribution_description": "uniquetitle", + "topic_category": "location", + "csw_insert_date": "2012-09-28T19:19:45.779", + "purpose": null, + "date": "2012-09-28T19:19:45.470", + "name": "mylayer", + "temporal_extent_end": "1991-10-01", + "data_quality_statement": null, + "language": "eng", + "keywords_region": "USA", + "maintenance_frequency": null, + "csw_anytext": "\n \n 7373d024-4273-4c6b-9a36-41878f4ce67b \n \n \n eng \n \n \n utf8 \n \n \n dataset \n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/ \n \n \n WWW:LINK-1.0-http--link \n \n \n GeoNode profile page \n \n \n \n \n \n \n \n pointOfContact \n \n \n \n \n 2012-09-28T19:19:45Z \n \n \n ISO 19115:2003 - Geographic information - Metadata \n \n \n ISO 19115:2003 \n \n \n \n \n \n \n \n 4326 \n \n \n EPSG \n \n \n 6.11 \n \n \n \n \n \n \n \n \n \n \n uniquetitle \n \n \n \n \n 2012-09-28T19:19:45Z \n \n \n publication \n \n \n \n \n \n \n \n mapDigital \n \n \n \n \n something here \n \n \n \n \n \n completed \n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Amylayer&width=20&version=1.1.1&bbox=-180%2C180%2C-90%2C90&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20 \n \n \n Thumbnail for 'uniquetitle' \n \n \n image/png \n \n \n \n \n \n \n \n ESRI Shapefile \n \n \n \n 1.0 \n \n \n \n \n \n \n \n \n populartag \n \n \n \n theme \n \n \n \n \n \n \n \n \n United States of America \n \n \n place \n \n \n \n \n \n \n \n copyright \n \n \n \n \n None \n \n \n eng \n \n \n utf8 \n \n \n \n location \n \n \n \n \n \n \n \n -180 \n \n \n 180 \n \n \n -90 \n \n \n 90 \n \n \n \n \n \n \n \n \n \n \n \n \n Oct. 1, 1990, midnight \n Oct. 1, 1991, midnight \n \n \n \n \n \n \n \n \n No information provided \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:mylayer \n \n \n WWW:LINK-1.0-http--link \n \n \n Online link to the 'uniquetitle' description on GeoNode \n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset \n \n \n \n \n \n \n \n \n \n \n ", + "supplemental_information": "No information provided", + "typename": "geonode:mylayer", + "csw_type": "dataset", + "workspace": "", + "metadata_uploaded": false, + "csw_wkt_geometry": "SRID=EPSG:4326;POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))", + "temporal_extent_start": "1990-10-01", + "constraints_use": "copyright" + } + }, + { + "pk": 4, + "model": "layers.layer", + "fields": { + "constraints_other": null, + "bbox_x0": "-180", + "csw_mdsource": "local", + "bbox_y1": "90", + "csw_typename": "gmd:MD_Metadata", + "date_type": "publication", + "srid": "EPSG:4326", + "bbox_x1": "180", + "metadata_xml": "\n \n 3e00ceb3-5848-475e-960f-1845f4ad9549\n \n \n eng\n \n \n utf8\n \n \n dataset\n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/\n \n \n WWW:LINK-1.0-http--link\n \n \n GeoNode profile page\n \n \n \n \n \n \n \n pointOfContact\n \n \n \n \n 2012-09-28T19:19:47Z\n \n \n ISO 19115:2003 - Geographic information - Metadata\n \n \n ISO 19115:2003\n \n \n \n \n \n \n \n 4326\n \n \n EPSG\n \n \n 6.11\n \n \n \n \n \n \n \n \n \n \n common blar\n \n \n \n \n 2012-09-28T19:19:47Z\n \n \n publication\n \n \n \n \n \n \n \n mapDigital\n \n \n \n \n lorem ipsum\n \n \n \n \n \n completed\n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Afoo&width=20&version=1.1.1&bbox=-180%2C180%2C-90%2C90&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20\n \n \n Thumbnail for 'common blar'\n \n \n image/png\n \n \n \n \n \n \n \n ESRI Shapefile\n \n \n \n 1.0\n \n \n \n \n \n \n \n \n populartag\n \n \n \n layertagunique\n \n \n \n theme\n \n \n \n \n \n \n \n \n United States of America\n \n \n place\n \n \n \n \n \n \n \n copyright\n \n \n \n \n None\n \n \n eng\n \n \n utf8\n \n \n \n location\n \n \n \n \n \n \n \n -180\n \n \n 180\n \n \n -90\n \n \n 90\n \n \n \n \n \n \n \n \n \n \n \n \n June 3, 1900, midnight\n June 3, 1901, midnight\n \n \n \n \n \n \n \n \n No information provided\n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:foo\n \n \n WWW:LINK-1.0-http--link\n \n \n Online link to the 'common blar' description on GeoNode\n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset\n \n \n \n \n \n \n \n \n \n \n \n", + "edition": null, + "owner": [ + "jblaze" + ], + "distribution_url": "http://localhost:8000/data/geonode:foo", + "spatial_representation_type": null, + "uuid": "3e00ceb3-5848-475e-960f-1845f4ad9549", + "title": "common blar", + "csw_schema": "http://www.isotc211.org/2005/gmd", + "storeType": "vector", + "abstract": "lorem ipsum", + "store": "", + "bbox_y0": "-90", + "distribution_description": "common blar", + "topic_category": "location", + "csw_insert_date": "2012-09-28T19:19:47.953", + "purpose": null, + "date": "2012-09-28T19:19:47.668", + "name": "foo", + "temporal_extent_end": "1901-06-03", + "data_quality_statement": null, + "language": "eng", + "keywords_region": "USA", + "maintenance_frequency": null, + "csw_anytext": "\n \n 3e00ceb3-5848-475e-960f-1845f4ad9549 \n \n \n eng \n \n \n utf8 \n \n \n dataset \n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/ \n \n \n WWW:LINK-1.0-http--link \n \n \n GeoNode profile page \n \n \n \n \n \n \n \n pointOfContact \n \n \n \n \n 2012-09-28T19:19:47Z \n \n \n ISO 19115:2003 - Geographic information - Metadata \n \n \n ISO 19115:2003 \n \n \n \n \n \n \n \n 4326 \n \n \n EPSG \n \n \n 6.11 \n \n \n \n \n \n \n \n \n \n \n common blar \n \n \n \n \n 2012-09-28T19:19:47Z \n \n \n publication \n \n \n \n \n \n \n \n mapDigital \n \n \n \n \n lorem ipsum \n \n \n \n \n \n completed \n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Afoo&width=20&version=1.1.1&bbox=-180%2C180%2C-90%2C90&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20 \n \n \n Thumbnail for 'common blar' \n \n \n image/png \n \n \n \n \n \n \n \n ESRI Shapefile \n \n \n \n 1.0 \n \n \n \n \n \n \n \n \n populartag \n \n \n \n layertagunique \n \n \n \n theme \n \n \n \n \n \n \n \n \n United States of America \n \n \n place \n \n \n \n \n \n \n \n copyright \n \n \n \n \n None \n \n \n eng \n \n \n utf8 \n \n \n \n location \n \n \n \n \n \n \n \n -180 \n \n \n 180 \n \n \n -90 \n \n \n 90 \n \n \n \n \n \n \n \n \n \n \n \n \n June 3, 1900, midnight \n June 3, 1901, midnight \n \n \n \n \n \n \n \n \n No information provided \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:foo \n \n \n WWW:LINK-1.0-http--link \n \n \n Online link to the 'common blar' description on GeoNode \n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset \n \n \n \n \n \n \n \n \n \n \n ", + "supplemental_information": "No information provided", + "typename": "geonode:foo", + "csw_type": "dataset", + "workspace": "", + "metadata_uploaded": false, + "csw_wkt_geometry": "SRID=EPSG:4326;POLYGON((-180 -90,-180 90,180 90,180 -90,-180 -90))", + "temporal_extent_start": "1900-06-03", + "constraints_use": "copyright" + } + }, + { + "pk": 5, + "model": "layers.layer", + "fields": { + "constraints_other": null, + "bbox_x0": "0", + "csw_mdsource": "local", + "bbox_y1": "1", + "csw_typename": "gmd:MD_Metadata", + "date_type": "publication", + "srid": "EPSG:4326", + "bbox_x1": "1", + "metadata_xml": "\n \n 33f457e3-1566-4c0d-abee-f1780f71781b\n \n \n eng\n \n \n utf8\n \n \n dataset\n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/\n \n \n WWW:LINK-1.0-http--link\n \n \n GeoNode profile page\n \n \n \n \n \n \n \n pointOfContact\n \n \n \n \n 2012-09-28T19:19:50Z\n \n \n ISO 19115:2003 - Geographic information - Metadata\n \n \n ISO 19115:2003\n \n \n \n \n \n \n \n 4326\n \n \n EPSG\n \n \n 6.11\n \n \n \n \n \n \n \n \n \n \n common double it\n \n \n \n \n 2012-09-28T19:19:50Z\n \n \n publication\n \n \n \n \n \n \n \n mapDigital\n \n \n \n \n whatever\n \n \n \n \n \n completed\n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Awhatever&width=20&version=1.1.1&bbox=0%2C1%2C0%2C1&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20\n \n \n Thumbnail for 'common double it'\n \n \n image/png\n \n \n \n \n \n \n \n ESRI Shapefile\n \n \n \n 1.0\n \n \n \n \n \n \n \n \n populartag\n \n \n \n theme\n \n \n \n \n \n \n \n \n United States of America\n \n \n place\n \n \n \n \n \n \n \n copyright\n \n \n \n \n None\n \n \n eng\n \n \n utf8\n \n \n \n location\n \n \n \n \n \n \n \n 0\n \n \n 1\n \n \n 0\n \n \n 1\n \n \n \n \n \n \n \n \n \n \n \n \n Nov. 1, 5000, midnight\n Nov. 1, 5001, midnight\n \n \n \n \n \n \n \n \n No information provided\n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:whatever\n \n \n WWW:LINK-1.0-http--link\n \n \n Online link to the 'common double it' description on GeoNode\n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset\n \n \n \n \n \n \n \n \n \n \n \n", + "edition": null, + "owner": [ + "foo" + ], + "distribution_url": "http://localhost:8000/data/geonode:whatever", + "spatial_representation_type": null, + "uuid": "33f457e3-1566-4c0d-abee-f1780f71781b", + "title": "common double it", + "csw_schema": "http://www.isotc211.org/2005/gmd", + "storeType": "raster", + "abstract": "whatever", + "store": "", + "bbox_y0": "0", + "distribution_description": "common double it", + "topic_category": "location", + "csw_insert_date": "2012-09-28T19:19:50.939", + "purpose": null, + "date": "2012-09-28T19:19:50.616", + "name": "whatever", + "temporal_extent_end": "5001-11-01", + "data_quality_statement": null, + "language": "eng", + "keywords_region": "USA", + "maintenance_frequency": null, + "csw_anytext": "\n \n 33f457e3-1566-4c0d-abee-f1780f71781b \n \n \n eng \n \n \n utf8 \n \n \n dataset \n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/ \n \n \n WWW:LINK-1.0-http--link \n \n \n GeoNode profile page \n \n \n \n \n \n \n \n pointOfContact \n \n \n \n \n 2012-09-28T19:19:50Z \n \n \n ISO 19115:2003 - Geographic information - Metadata \n \n \n ISO 19115:2003 \n \n \n \n \n \n \n \n 4326 \n \n \n EPSG \n \n \n 6.11 \n \n \n \n \n \n \n \n \n \n \n common double it \n \n \n \n \n 2012-09-28T19:19:50Z \n \n \n publication \n \n \n \n \n \n \n \n mapDigital \n \n \n \n \n whatever \n \n \n \n \n \n completed \n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Awhatever&width=20&version=1.1.1&bbox=0%2C1%2C0%2C1&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20 \n \n \n Thumbnail for 'common double it' \n \n \n image/png \n \n \n \n \n \n \n \n ESRI Shapefile \n \n \n \n 1.0 \n \n \n \n \n \n \n \n \n populartag \n \n \n \n theme \n \n \n \n \n \n \n \n \n United States of America \n \n \n place \n \n \n \n \n \n \n \n copyright \n \n \n \n \n None \n \n \n eng \n \n \n utf8 \n \n \n \n location \n \n \n \n \n \n \n \n 0 \n \n \n 1 \n \n \n 0 \n \n \n 1 \n \n \n \n \n \n \n \n \n \n \n \n \n Nov. 1, 5000, midnight \n Nov. 1, 5001, midnight \n \n \n \n \n \n \n \n \n No information provided \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:whatever \n \n \n WWW:LINK-1.0-http--link \n \n \n Online link to the 'common double it' description on GeoNode \n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset \n \n \n \n \n \n \n \n \n \n \n ", + "supplemental_information": "No information provided", + "typename": "geonode:whatever", + "csw_type": "dataset", + "workspace": "", + "metadata_uploaded": false, + "csw_wkt_geometry": "SRID=EPSG:4326;POLYGON((0 0,0 1,1 1,1 0,0 0))", + "temporal_extent_start": "5000-11-01", + "constraints_use": "copyright" + } + }, + { + "pk": 6, + "model": "layers.layer", + "fields": { + "constraints_other": null, + "bbox_x0": "0", + "csw_mdsource": "local", + "bbox_y1": "5", + "csw_typename": "gmd:MD_Metadata", + "date_type": "publication", + "srid": "EPSG:4326", + "bbox_x1": "5", + "metadata_xml": "\n \n c5a70533-aa3e-47c6-ad65-11435eeb22a8\n \n \n eng\n \n \n utf8\n \n \n dataset\n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/\n \n \n WWW:LINK-1.0-http--link\n \n \n GeoNode profile page\n \n \n \n \n \n \n \n pointOfContact\n \n \n \n \n 2012-09-28T19:19:52Z\n \n \n ISO 19115:2003 - Geographic information - Metadata\n \n \n ISO 19115:2003\n \n \n \n \n \n \n \n 4326\n \n \n EPSG\n \n \n 6.11\n \n \n \n \n \n \n \n \n \n \n common double time\n \n \n \n \n 2012-09-28T19:19:52Z\n \n \n publication\n \n \n \n \n \n \n \n mapDigital\n \n \n \n \n else\n \n \n \n \n \n completed\n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Afooey&width=20&version=1.1.1&bbox=0%2C5%2C0%2C5&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20\n \n \n Thumbnail for 'common double time'\n \n \n image/png\n \n \n \n \n \n \n \n ESRI Shapefile\n \n \n \n 1.0\n \n \n \n \n \n \n \n \n populartag\n \n \n \n theme\n \n \n \n \n \n \n \n \n United States of America\n \n \n place\n \n \n \n \n \n \n \n copyright\n \n \n \n \n None\n \n \n eng\n \n \n utf8\n \n \n \n location\n \n \n \n \n \n \n \n 0\n \n \n 5\n \n \n 0\n \n \n 5\n \n \n \n \n \n \n \n \n \n \n \n \n Jan. 1, 1, midnight\n Jan. 1, 2, midnight\n \n \n \n \n \n \n \n \n No information provided\n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:fooey\n \n \n WWW:LINK-1.0-http--link\n \n \n Online link to the 'common double time' description on GeoNode\n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset\n \n \n \n \n \n \n \n \n \n \n \n", + "edition": null, + "owner": [ + "user1" + ], + "distribution_url": "http://localhost:8000/data/geonode:fooey", + "spatial_representation_type": null, + "uuid": "c5a70533-aa3e-47c6-ad65-11435eeb22a8", + "title": "common double time", + "csw_schema": "http://www.isotc211.org/2005/gmd", + "storeType": "vector", + "abstract": "else", + "store": "", + "bbox_y0": "0", + "distribution_description": "common double time", + "topic_category": "location", + "csw_insert_date": "2012-09-28T19:19:53.240", + "purpose": null, + "date": "2012-09-28T19:19:52.907", + "name": "fooey", + "temporal_extent_end": "0002-01-01", + "data_quality_statement": null, + "language": "eng", + "keywords_region": "USA", + "maintenance_frequency": null, + "csw_anytext": "\n \n c5a70533-aa3e-47c6-ad65-11435eeb22a8 \n \n \n eng \n \n \n utf8 \n \n \n dataset \n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/ \n \n \n WWW:LINK-1.0-http--link \n \n \n GeoNode profile page \n \n \n \n \n \n \n \n pointOfContact \n \n \n \n \n 2012-09-28T19:19:52Z \n \n \n ISO 19115:2003 - Geographic information - Metadata \n \n \n ISO 19115:2003 \n \n \n \n \n \n \n \n 4326 \n \n \n EPSG \n \n \n 6.11 \n \n \n \n \n \n \n \n \n \n \n common double time \n \n \n \n \n 2012-09-28T19:19:52Z \n \n \n publication \n \n \n \n \n \n \n \n mapDigital \n \n \n \n \n else \n \n \n \n \n \n completed \n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Afooey&width=20&version=1.1.1&bbox=0%2C5%2C0%2C5&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20 \n \n \n Thumbnail for 'common double time' \n \n \n image/png \n \n \n \n \n \n \n \n ESRI Shapefile \n \n \n \n 1.0 \n \n \n \n \n \n \n \n \n populartag \n \n \n \n theme \n \n \n \n \n \n \n \n \n United States of America \n \n \n place \n \n \n \n \n \n \n \n copyright \n \n \n \n \n None \n \n \n eng \n \n \n utf8 \n \n \n \n location \n \n \n \n \n \n \n \n 0 \n \n \n 5 \n \n \n 0 \n \n \n 5 \n \n \n \n \n \n \n \n \n \n \n \n \n Jan. 1, 1, midnight \n Jan. 1, 2, midnight \n \n \n \n \n \n \n \n \n No information provided \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:fooey \n \n \n WWW:LINK-1.0-http--link \n \n \n Online link to the 'common double time' description on GeoNode \n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset \n \n \n \n \n \n \n \n \n \n \n ", + "supplemental_information": "No information provided", + "typename": "geonode:fooey", + "csw_type": "dataset", + "workspace": "", + "metadata_uploaded": false, + "csw_wkt_geometry": "SRID=EPSG:4326;POLYGON((0 0,0 5,5 5,5 0,0 0))", + "temporal_extent_start": "0001-01-01", + "constraints_use": "copyright" + } + }, + { + "pk": 7, + "model": "layers.layer", + "fields": { + "constraints_other": null, + "bbox_x0": "0", + "csw_mdsource": "local", + "bbox_y1": "10", + "csw_typename": "gmd:MD_Metadata", + "date_type": "publication", + "srid": "EPSG:4326", + "bbox_x1": "10", + "metadata_xml": "\n \n 49c815a0-8d5e-4c7d-812f-c125185a9a16\n \n \n eng\n \n \n utf8\n \n \n dataset\n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/\n \n \n WWW:LINK-1.0-http--link\n \n \n GeoNode profile page\n \n \n \n \n \n \n \n pointOfContact\n \n \n \n \n 2012-09-28T19:19:55Z\n \n \n ISO 19115:2003 - Geographic information - Metadata\n \n \n ISO 19115:2003\n \n \n \n \n \n \n \n 4326\n \n \n EPSG\n \n \n 6.11\n \n \n \n \n \n \n \n \n \n \n common bar\n \n \n \n \n 2012-09-28T19:19:55Z\n \n \n publication\n \n \n \n \n \n \n \n mapDigital\n \n \n \n \n uniqueabstract\n \n \n \n \n \n completed\n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Aquux&width=20&version=1.1.1&bbox=0%2C10%2C0%2C10&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20\n \n \n Thumbnail for 'common bar'\n \n \n image/png\n \n \n \n \n \n \n \n ESRI Shapefile\n \n \n \n 1.0\n \n \n \n \n \n \n \n \n populartag\n \n \n \n theme\n \n \n \n \n \n \n \n \n United States of America\n \n \n place\n \n \n \n \n \n \n \n copyright\n \n \n \n \n None\n \n \n eng\n \n \n utf8\n \n \n \n location\n \n \n \n \n \n \n \n 0\n \n \n 10\n \n \n 0\n \n \n 10\n \n \n \n \n \n \n \n \n \n \n \n \n Dec. 9, 1950, midnight\n Dec. 9, 1951, midnight\n \n \n \n \n \n \n \n \n No information provided\n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:quux\n \n \n WWW:LINK-1.0-http--link\n \n \n Online link to the 'common bar' description on GeoNode\n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset\n \n \n \n \n \n \n \n \n \n \n \n", + "edition": null, + "owner": [ + "user2" + ], + "distribution_url": "http://localhost:8000/data/geonode:quux", + "spatial_representation_type": null, + "uuid": "49c815a0-8d5e-4c7d-812f-c125185a9a16", + "title": "common bar", + "csw_schema": "http://www.isotc211.org/2005/gmd", + "storeType": "raster", + "abstract": "uniqueabstract", + "store": "", + "bbox_y0": "0", + "distribution_description": "common bar", + "topic_category": "location", + "csw_insert_date": "2012-09-28T19:19:55.536", + "purpose": null, + "date": "2012-09-28T19:19:55.213", + "name": "quux", + "temporal_extent_end": "1951-12-09", + "data_quality_statement": null, + "language": "eng", + "keywords_region": "USA", + "maintenance_frequency": null, + "csw_anytext": "\n \n 49c815a0-8d5e-4c7d-812f-c125185a9a16 \n \n \n eng \n \n \n utf8 \n \n \n dataset \n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/ \n \n \n WWW:LINK-1.0-http--link \n \n \n GeoNode profile page \n \n \n \n \n \n \n \n pointOfContact \n \n \n \n \n 2012-09-28T19:19:55Z \n \n \n ISO 19115:2003 - Geographic information - Metadata \n \n \n ISO 19115:2003 \n \n \n \n \n \n \n \n 4326 \n \n \n EPSG \n \n \n 6.11 \n \n \n \n \n \n \n \n \n \n \n common bar \n \n \n \n \n 2012-09-28T19:19:55Z \n \n \n publication \n \n \n \n \n \n \n \n mapDigital \n \n \n \n \n uniqueabstract \n \n \n \n \n \n completed \n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Aquux&width=20&version=1.1.1&bbox=0%2C10%2C0%2C10&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20 \n \n \n Thumbnail for 'common bar' \n \n \n image/png \n \n \n \n \n \n \n \n ESRI Shapefile \n \n \n \n 1.0 \n \n \n \n \n \n \n \n \n populartag \n \n \n \n theme \n \n \n \n \n \n \n \n \n United States of America \n \n \n place \n \n \n \n \n \n \n \n copyright \n \n \n \n \n None \n \n \n eng \n \n \n utf8 \n \n \n \n location \n \n \n \n \n \n \n \n 0 \n \n \n 10 \n \n \n 0 \n \n \n 10 \n \n \n \n \n \n \n \n \n \n \n \n \n Dec. 9, 1950, midnight \n Dec. 9, 1951, midnight \n \n \n \n \n \n \n \n \n No information provided \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:quux \n \n \n WWW:LINK-1.0-http--link \n \n \n Online link to the 'common bar' description on GeoNode \n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset \n \n \n \n \n \n \n \n \n \n \n ", + "supplemental_information": "No information provided", + "typename": "geonode:quux", + "csw_type": "dataset", + "workspace": "", + "metadata_uploaded": false, + "csw_wkt_geometry": "SRID=EPSG:4326;POLYGON((0 0,0 10,10 10,10 0,0 0))", + "temporal_extent_start": "1950-12-09", + "constraints_use": "copyright" + } + }, + { + "pk": 8, + "model": "layers.layer", + "fields": { + "constraints_other": null, + "bbox_x0": "0", + "csw_mdsource": "local", + "bbox_y1": "50", + "csw_typename": "gmd:MD_Metadata", + "date_type": "publication", + "srid": "EPSG:4326", + "bbox_x1": "50", + "metadata_xml": "\n \n 5da8e574-7dda-4d1f-82c7-57168ec5c2c9\n \n \n eng\n \n \n utf8\n \n \n dataset\n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/\n \n \n WWW:LINK-1.0-http--link\n \n \n GeoNode profile page\n \n \n \n \n \n \n \n pointOfContact\n \n \n \n \n 2012-09-28T19:19:57Z\n \n \n ISO 19115:2003 - Geographic information - Metadata\n \n \n ISO 19115:2003\n \n \n \n \n \n \n \n 4326\n \n \n EPSG\n \n \n 6.11\n \n \n \n \n \n \n \n \n \n \n common morx\n \n \n \n \n 2012-09-28T19:19:57Z\n \n \n publication\n \n \n \n \n \n \n \n mapDigital\n \n \n \n \n lorem ipsum\n \n \n \n \n \n completed\n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Afleem&width=20&version=1.1.1&bbox=0%2C50%2C0%2C50&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20\n \n \n Thumbnail for 'common morx'\n \n \n image/png\n \n \n \n \n \n \n \n ESRI Shapefile\n \n \n \n 1.0\n \n \n \n \n \n \n \n \n populartag\n \n \n \n theme\n \n \n \n \n \n \n \n \n United States of America\n \n \n place\n \n \n \n \n \n \n \n copyright\n \n \n \n \n None\n \n \n eng\n \n \n utf8\n \n \n \n location\n \n \n \n \n \n \n \n 0\n \n \n 50\n \n \n 0\n \n \n 50\n \n \n \n \n \n \n \n \n \n \n \n \n Aug. 29, 1963, midnight\n Aug. 28, 1964, midnight\n \n \n \n \n \n \n \n \n No information provided\n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:fleem\n \n \n WWW:LINK-1.0-http--link\n \n \n Online link to the 'common morx' description on GeoNode\n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset\n \n \n \n \n \n \n \n \n \n \n \n", + "edition": null, + "owner": [ + "unique_username" + ], + "distribution_url": "http://localhost:8000/data/geonode:fleem", + "spatial_representation_type": null, + "uuid": "5da8e574-7dda-4d1f-82c7-57168ec5c2c9", + "title": "common morx", + "csw_schema": "http://www.isotc211.org/2005/gmd", + "storeType": "vector", + "abstract": "lorem ipsum", + "store": "", + "bbox_y0": "0", + "distribution_description": "common morx", + "topic_category": "location", + "csw_insert_date": "2012-09-28T19:19:57.911", + "purpose": null, + "date": "2012-09-28T19:19:57.569", + "name": "fleem", + "temporal_extent_end": "1964-08-28", + "data_quality_statement": null, + "language": "eng", + "keywords_region": "USA", + "maintenance_frequency": null, + "csw_anytext": "\n \n 5da8e574-7dda-4d1f-82c7-57168ec5c2c9 \n \n \n eng \n \n \n utf8 \n \n \n dataset \n \n \n \n \n admin \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/profiles/profile/admin/ \n \n \n WWW:LINK-1.0-http--link \n \n \n GeoNode profile page \n \n \n \n \n \n \n \n pointOfContact \n \n \n \n \n 2012-09-28T19:19:57Z \n \n \n ISO 19115:2003 - Geographic information - Metadata \n \n \n ISO 19115:2003 \n \n \n \n \n \n \n \n 4326 \n \n \n EPSG \n \n \n 6.11 \n \n \n \n \n \n \n \n \n \n \n common morx \n \n \n \n \n 2012-09-28T19:19:57Z \n \n \n publication \n \n \n \n \n \n \n \n mapDigital \n \n \n \n \n lorem ipsum \n \n \n \n \n \n completed \n \n \n \n \n \n http://localhost:8080/geoserver/wms?layers=geonode%3Afleem&width=20&version=1.1.1&bbox=0%2C50%2C0%2C50&service=WMS&format=image%2Fpng&srs=EPSG%3A4326&request=GetMap&height=20 \n \n \n Thumbnail for 'common morx' \n \n \n image/png \n \n \n \n \n \n \n \n ESRI Shapefile \n \n \n \n 1.0 \n \n \n \n \n \n \n \n \n populartag \n \n \n \n theme \n \n \n \n \n \n \n \n \n United States of America \n \n \n place \n \n \n \n \n \n \n \n copyright \n \n \n \n \n None \n \n \n eng \n \n \n utf8 \n \n \n \n location \n \n \n \n \n \n \n \n 0 \n \n \n 50 \n \n \n 0 \n \n \n 50 \n \n \n \n \n \n \n \n \n \n \n \n \n Aug. 29, 1963, midnight \n Aug. 28, 1964, midnight \n \n \n \n \n \n \n \n \n No information provided \n \n \n \n \n \n \n \n \n \n \n http://localhost:8000/data/geonode:fleem \n \n \n WWW:LINK-1.0-http--link \n \n \n Online link to the 'common morx' description on GeoNode \n \n \n \n \n \n \n \n \n \n \n \n \n \n dataset \n \n \n \n \n \n \n \n \n \n \n ", + "supplemental_information": "No information provided", + "typename": "geonode:fleem", + "csw_type": "dataset", + "workspace": "", + "metadata_uploaded": false, + "csw_wkt_geometry": "SRID=EPSG:4326;POLYGON((0 0,0 50,50 50,50 0,0 0))", + "temporal_extent_start": "1963-08-29", + "constraints_use": "copyright" + } + }, + { + "pk": 1, + "model": "maps.map", + "fields": { + "projection": "EPSG:4326", + "title": "lorem ipsum", + "abstract": "common lorem ipsum", + "zoom": 4, + "center_x": 42.0, + "center_y": -73.0, + "last_modified": "2012-09-28T19:19:35.529", + "owner": [ + "user1" + ] + } + }, + { + "pk": 2, + "model": "maps.map", + "fields": { + "projection": "EPSG:4326", + "title": "ipsum lorem", + "abstract": "common ipsum lorem", + "zoom": 4, + "center_x": 42.0, + "center_y": -73.0, + "last_modified": "2012-09-28T19:19:36.095", + "owner": [ + "user2" + ] + } + }, + { + "pk": 3, + "model": "maps.map", + "fields": { + "projection": "EPSG:4326", + "title": "lorem1 ipsum1", + "abstract": "common abstract1", + "zoom": 4, + "center_x": 42.0, + "center_y": -73.0, + "last_modified": "2012-09-28T19:19:36.957", + "owner": [ + "unique_username" + ] + } + }, + { + "pk": 4, + "model": "maps.map", + "fields": { + "projection": "EPSG:4326", + "title": "ipsum foo", + "abstract": "common bar lorem", + "zoom": 4, + "center_x": 42.0, + "center_y": -73.0, + "last_modified": "2012-09-28T19:19:37.355", + "owner": [ + "jblaze" + ] + } + }, + { + "pk": 5, + "model": "maps.map", + "fields": { + "projection": "EPSG:4326", + "title": "map one", + "abstract": "common this is a unique thing", + "zoom": 4, + "center_x": 42.0, + "center_y": -73.0, + "last_modified": "2012-09-28T19:19:38.036", + "owner": [ + "foo" + ] + } + }, + { + "pk": 6, + "model": "maps.map", + "fields": { + "projection": "EPSG:4326", + "title": "quux", + "abstract": "common double thing", + "zoom": 4, + "center_x": 42.0, + "center_y": -73.0, + "last_modified": "2012-09-28T19:19:38.459", + "owner": [ + "user1" + ] + } + }, + { + "pk": 7, + "model": "maps.map", + "fields": { + "projection": "EPSG:4326", + "title": "morx", + "abstract": "common thing double", + "zoom": 4, + "center_x": 42.0, + "center_y": -73.0, + "last_modified": "2012-09-28T19:19:38.873", + "owner": [ + "user2" + ] + } + }, + { + "pk": 8, + "model": "maps.map", + "fields": { + "projection": "EPSG:4326", + "title": "titledupe something else ", + "abstract": "whatever common", + "zoom": 4, + "center_x": 42.0, + "center_y": -73.0, + "last_modified": "2012-09-28T19:19:39.280", + "owner": [ + "unique_username" + ] + } + }, + { + "pk": 9, + "model": "maps.map", + "fields": { + "projection": "EPSG:4326", + "title": "something titledupe else ", + "abstract": "bar common", + "zoom": 4, + "center_x": 42.0, + "center_y": -73.0, + "last_modified": "2012-09-28T19:19:39.762", + "owner": [ + "jblaze" + ] + } + }, + { + "pk": 1, + "model": "taggit.tag", + "fields": { + "name": "populartag", + "slug": "populartag" + } + }, + { + "pk": 2, + "model": "taggit.tag", + "fields": { + "name": "maptagunique", + "slug": "maptagunique" + } + }, + { + "pk": 3, + "model": "taggit.tag", + "fields": { + "name": "here", + "slug": "here" + } + }, + { + "pk": 4, + "model": "taggit.tag", + "fields": { + "name": "layertagunique", + "slug": "layertagunique" + } + }, + { + "pk": 1, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "maps", + "map" + ], + "object_id": 1 + } + }, + { + "pk": 2, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "maps", + "map" + ], + "object_id": 2 + } + }, + { + "pk": 3, + "model": "taggit.taggeditem", + "fields": { + "tag": 2, + "content_type": [ + "maps", + "map" + ], + "object_id": 2 + } + }, + { + "pk": 4, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "maps", + "map" + ], + "object_id": 3 + } + }, + { + "pk": 5, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "maps", + "map" + ], + "object_id": 4 + } + }, + { + "pk": 6, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "maps", + "map" + ], + "object_id": 5 + } + }, + { + "pk": 7, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "maps", + "map" + ], + "object_id": 6 + } + }, + { + "pk": 8, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "maps", + "map" + ], + "object_id": 7 + } + }, + { + "pk": 9, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "maps", + "map" + ], + "object_id": 8 + } + }, + { + "pk": 10, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "maps", + "map" + ], + "object_id": 9 + } + }, + { + "pk": 11, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "layers", + "layer" + ], + "object_id": 1 + } + }, + { + "pk": 12, + "model": "taggit.taggeditem", + "fields": { + "tag": 3, + "content_type": [ + "layers", + "layer" + ], + "object_id": 1 + } + }, + { + "pk": 13, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "layers", + "layer" + ], + "object_id": 2 + } + }, + { + "pk": 14, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "layers", + "layer" + ], + "object_id": 3 + } + }, + { + "pk": 15, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "layers", + "layer" + ], + "object_id": 4 + } + }, + { + "pk": 16, + "model": "taggit.taggeditem", + "fields": { + "tag": 4, + "content_type": [ + "layers", + "layer" + ], + "object_id": 4 + } + }, + { + "pk": 17, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "layers", + "layer" + ], + "object_id": 5 + } + }, + { + "pk": 18, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "layers", + "layer" + ], + "object_id": 6 + } + }, + { + "pk": 19, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "layers", + "layer" + ], + "object_id": 7 + } + }, + { + "pk": 20, + "model": "taggit.taggeditem", + "fields": { + "tag": 1, + "content_type": [ + "layers", + "layer" + ], + "object_id": 8 + } + } +] \ No newline at end of file diff --git a/geonode/search/backends/silage/geomodels.py b/geonode/search/backends/silage/geomodels.py new file mode 100644 index 00000000000..16bbae88fbc --- /dev/null +++ b/geonode/search/backends/silage/geomodels.py @@ -0,0 +1,221 @@ +######################################################################### +# +# Copyright (C) 2012 OpenPlans +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.contrib.gis.db import models +from django.contrib.gis.gdal import Envelope + +from django.db.models import signals + +from geonode.maps.models import Layer +from geonode.maps.models import Map +from geonode.maps.models import map_changed_signal +from geonode.search.backends.silage import util + +from logging import getLogger + +_logger = getLogger(__name__) + +class SpatialTemporalIndex(models.Model): + time_start = models.BigIntegerField(null=True) + time_end = models.BigIntegerField(null=True) + extent = models.PolygonField() + objects = models.GeoManager() + + class Meta: + abstract = True + + def __unicode__(self): + return ' for %s, %s, %s - %s' % ( + self.indexed, + self.extent.extent, + util.jdate_to_approx_iso_str(self.time_start), + util.jdate_to_approx_iso_str(self.time_end) + ) + +class LayerIndex(SpatialTemporalIndex): + indexed = models.OneToOneField(Layer,related_name='spatial_temporal_index') + +class MapIndex(SpatialTemporalIndex): + indexed = models.OneToOneField(Map,related_name='spatial_temporal_index') + + +def _index_for_model(model): + if model == Layer: + index = LayerIndex + elif model == Map: + index = MapIndex + else: + raise Exception('no index for model', model) + return index + + +def filter_by_extent(model, q, extent, user=None): + env = Envelope(extent) + index = _index_for_model(model) + extent_ids = index.objects.filter(extent__contained=env.wkt) + if user: + extent_ids = extent_ids.values('indexed__owner') + return q.filter(user__in=extent_ids) + else: + extent_ids = extent_ids.values('indexed') + return q.filter(id__in=extent_ids) + + +def filter_by_period(model, q, start, end, user=None): + index = _index_for_model(model) + q = index.objects.all() + if start: + q = q.filter(time_start__gte = util.iso_str_to_jdate(start)) + if end: + q = q.filter(time_end__lte = util.iso_str_to_jdate(end)) + if user: + period_ids = q.values('indexed__owner') + q = q.filter(user__in=period_ids) + else: + period_ids = q.values('indexed') + q = q.filter(id__in=period_ids) + return q + + +def index_object(obj, update=False): + if type(obj) == Layer: + index = LayerIndex + func = index_layer + elif type(obj) == Map: + index = MapIndex + func = index_map + else: + raise Exception('cannot index %s' % obj) + + created = False + try: + index_obj = index.objects.get(indexed=obj) + except index.DoesNotExist: + _logger.debug('created index for %s',obj) + index_obj = index(indexed=obj) + created = True + + if not update or created: + _logger.debug('indexing %s',obj) + try: + func(index_obj, obj) + except: + _logger.exception('Error indexing object %s', obj) + else: + _logger.debug('skipping %s',obj) + + +def index_layer(index, obj): + start = end = None + try: + start, end = obj.get_time_extent() + except: + _logger.warn('could not get time info for %s', obj.typename) + + if start: + index.time_start = util.iso_str_to_jdate(start) + if end: + index.time_end = util.iso_str_to_jdate(end) + + try: + wms_metadata = obj.metadata() + except: + _logger.warn('could not get WMS info for %s', obj.typename) + return + + min_x, min_y, max_x, max_y = wms_metadata.boundingBoxWGS84 + + if wms_metadata.boundingBoxWGS84 != (0.0,0.0,-1.0,-1.0): + try: + index.extent = Envelope(min_x,min_y,max_x,max_y).wkt; + except Exception,ex: + _logger.warn('Error computing envelope: %s, bounding box was %s', str(ex),wms_metadata.boundingBoxWGS84) + else: + #@todo might be better to have a nullable extent + _logger.warn('Bounding box empty, adding default envelope') + index.extent = Envelope(-180,-90,180,90).wkt + + index.save() + + +def index_map(index, obj): + time_start = None + time_end = None + extent = Envelope(0,0,0,0) + for l in obj.local_layers: + start = end = None + try: + start, end = l.get_time_extent() + except: + _logger.warn('could not get time info for %s', l.typename) + + if start: + start = util.iso_str_to_jdate(start) + if time_start is None: + time_start = start + else: + time_start = min(time_start, start) + if end: + end = util.iso_str_to_jdate(end) + if time_end is None: + time_end = start + else: + time_end = max(time_end, end) + + try: + wms_metadata = l.metadata() + extent.expand_to_include(wms_metadata.boundingBoxWGS84) + except: + _logger.warn('could not get WMS info for %s', l.typename ) + + if time_start: + index.time_start = time_start + if time_end: + index.time_end = time_end + index.extent = extent.wkt + index.save() + + +def object_created(instance, sender, **kw): + if kw['created']: + index_object(instance) + + +def map_updated(sender, **kw): + if kw['what_changed'] == 'layers': + index_object(sender) + + +def object_deleted(instance, sender, **kw): + if type(instance) == Layer: + index = LayerIndex + elif type(instance) == Map: + index = MapIndex + try: + index.objects.get(indexed=instance).delete() + except index.DoesNotExist: + pass + + +signals.post_save.connect(object_created, sender=Layer) + +signals.pre_delete.connect(object_deleted, sender=Map) +signals.pre_delete.connect(object_deleted, sender=Layer) + +map_changed_signal.connect(map_updated) \ No newline at end of file diff --git a/geonode/search/backends/silage/management/__init__.py b/geonode/search/backends/silage/management/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/search/backends/silage/management/commands/__init__.py b/geonode/search/backends/silage/management/commands/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/geonode/search/backends/silage/management/commands/update_spatial_temporal_indices.py b/geonode/search/backends/silage/management/commands/update_spatial_temporal_indices.py new file mode 100644 index 00000000000..053c082e9c4 --- /dev/null +++ b/geonode/search/backends/silage/management/commands/update_spatial_temporal_indices.py @@ -0,0 +1,52 @@ +######################################################################### +# +# Copyright (C) 2012 OpenPlans +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.core.management.base import BaseCommand +from geonode.maps.models import Map +from geonode.maps.models import Layer +from geonode.search.backends.silage.models import index_object +import logging +from optparse import make_option +import traceback +import signal +import sys + +def _handler(s,f): + sys.exit(0) +signal.signal(signal.SIGINT,_handler) + +class Command(BaseCommand): + help = 'Update silage indices' + option_list = BaseCommand.option_list + ( + make_option('--update', dest="update", default=False, action="store_true", + help="Update any existing entries"), + ) + + def handle(self, *args, **opts): + logging.getLogger('geonode.search.backends.silage.models').setLevel(logging.DEBUG) + update = opts['update'] + def index(o): + try: + index_object(o,update=update) + except Exception: + print 'error indexing', o + traceback.print_exc() + + map(index,Map.objects.all()) + map(index,Layer.objects.all()) \ No newline at end of file diff --git a/geonode/search/backends/silage/models.py b/geonode/search/backends/silage/models.py new file mode 100644 index 00000000000..bad8ce4cbc0 --- /dev/null +++ b/geonode/search/backends/silage/models.py @@ -0,0 +1,60 @@ +######################################################################### +# +# Copyright (C) 2012 OpenPlans +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.conf import settings + +from geonode.layers.models import Layer +from geonode.layers.models import add_bbox_query +from geonode.search.backends.silage.util import iso_fmt + +from datetime import datetime + +def filter_by_period(model, q, start, end, user=None): + '''modify the query to filter the given model for dates between start and end + start, end - iso str ('-5000-01-01T12:00:00Z') + ''' + parse = lambda v: datetime.strptime(v, iso_fmt) + if model == Layer and not user: + if start: + q = q.filter(temporal_extent_start__gte = parse(start)) + if end: + q = q.filter(temporal_extent_end__lte = parse(end)) + else: + # @todo handle map and/or users - either directly if implemented or ... + # this will effectively short-circuit the query at this point + q = q.none() + return q + +def filter_by_extent(model, q, extent, user=None): + '''modify the query to filter the given model for the provided extent and optional user + extent: tuple of float coordinates representing x0,x1,y0,y1 + ''' + if model == Layer and not user: + q = add_bbox_query(q, extent) + else: + # @todo handle map and/or users - either directly if implemented or ... + # this will effectively short-circuit the query at this point + q = q.none() + return q + +using_geodjango = False + +if 'django.contrib.gis' in settings.INSTALLED_APPS: + from geonode.search.backends.silage.geomodels import * + using_geodjango = True \ No newline at end of file diff --git a/geonode/search/backends/silage/normalizers.py b/geonode/search/backends/silage/normalizers.py new file mode 100644 index 00000000000..ccc9aa0b8de --- /dev/null +++ b/geonode/search/backends/silage/normalizers.py @@ -0,0 +1,202 @@ +from django.core.cache import cache +from django.contrib.contenttypes.models import ContentType +from django.core.exceptions import ObjectDoesNotExist +from django.core.urlresolvers import reverse +from django.template import defaultfilters + +from geonode.maps.models import Layer +from geonode.maps.models import Map +from geonode.search.backends.silage import extension + +from agon_ratings.categories import RATING_CATEGORY_LOOKUP +from agon_ratings.models import OverallRating + +from avatar.util import get_default_avatar_url + +_default_avatar_url = get_default_avatar_url() + +def _bbox(obj): + idx = None + # the one-to-one reverse relationship requires special handling if null :( + # maybe one day: https://code.djangoproject.com/ticket/10227 + try: + idx = obj.spatial_temporal_index + except ObjectDoesNotExist: + pass + except AttributeError: + pass + # unknown extent, just give something that works + extent = idx.extent.extent if idx else (-180,-90,180,90) + return dict(minx=extent[0], miny=extent[1], maxx=extent[2], maxy=extent[3]) + + +def _get_ratings(model): + '''cached, bulk access for a given models rating''' + key = 'overall_rating_%s' % model.__name__ + results = cache.get(key) + if not results: + # this is some hacky stuff related to rankings + choice = model.__name__.lower() + category = RATING_CATEGORY_LOOKUP.get( + "%s.%s-%s" % (model._meta.app_label, model._meta.object_name, choice) + ) + try: + ct = ContentType.objects.get_for_model(model) + ratings = OverallRating.objects.filter( + content_type = ct, + category = category + ) + results = dict([ (r.object_id, r.rating) or 0 for r in ratings]) + cache.set(key, results) + except OverallRating.DoesNotExist: + return {} + return results + + +def _annotate(normalizers): + '''annotate normalizers with any attributes that are better fetched in bulk''' + if not normalizers: return + model = type(normalizers[0].o) + ratings = _get_ratings(model) + for n in normalizers: + n.rating = float(ratings.get(n.o.id, 0)) + + +def apply_normalizers(results): + '''build the appropriate normalizers for the query set(s) and annotate''' + normalized = [] + mapping = [ + ('maps', MapNormalizer), + ('layers', LayerNormalizer), + ('owners', OwnerNormalizer), + ] + for k,n in mapping: + r = results.get(k, None) + if not r: continue + normalizers = map(n, r) + _annotate(normalizers) + extension.process_results(normalizers) + normalized.extend(normalizers) + return normalized + + +class Normalizer: + '''Base class to allow lazy normalization of Map and Layer attributes. + + The fields we support sorting on are rank, title, last_modified. + Instead of storing these (to keep pickle query size small), expose via methods. + ''' + def __init__(self,o,data = None): + self.o = o + self.data = data + self.dict = None + def rank(self): + return self.rating + def title(self): + return self.o.title + def last_modified(self): + raise Exception('abstract') + def relevance(self): + return getattr(self.o, 'relevance', 0) + def as_dict(self, exclude): + if self.dict is None: + if self.o._deferred: + self.o = getattr(type(self.o),'objects').get(pk = self.o.pk) + self.dict = self.populate(self.data or {}, exclude) + self.dict['iid'] = self.iid + self.dict['relevance'] = getattr(self.o, 'relevance', 0) + if hasattr(self,'views'): + self.dict['views'] = self.views + self.dict['rating'] = self.rating + if exclude: + for e in exclude: + if e in self.dict: self.dict.pop(e) + return self.dict + + +class MapNormalizer(Normalizer): + def last_modified(self): + return self.o.last_modified + def populate(self, doc, exclude): + mapobj = self.o + # resolve any local layers and their keywords + local_kw = [ l.keyword_list() for l in mapobj.local_layers if l.keywords] + keywords = local_kw and list(set( reduce(lambda a,b: a+b, local_kw))) or [] + doc['id'] = mapobj.id + doc['title'] = mapobj.title + doc['abstract'] = defaultfilters.linebreaks(mapobj.abstract) + doc['topic'] = '', # @todo + doc['detail'] = reverse('map_detail', args=(mapobj.id,)) + doc['owner'] = mapobj.owner.username +# doc['owner_detail'] = reverse('about_storyteller', args=(map.owner.username,)) + doc['owner_detail'] = mapobj.owner.get_absolute_url() + doc['last_modified'] = extension.date_fmt(mapobj.last_modified) + doc['_type'] = 'map' + doc['_display_type'] = extension.MAP_DISPLAY +# doc['thumb'] = map.get_thumbnail_url() + doc['keywords'] = keywords + if 'bbox' not in exclude: + doc['bbox'] = _bbox(mapobj) + return doc + + +class LayerNormalizer(Normalizer): + def last_modified(self): + return self.o.date + def populate(self, doc, exclude): + layer = self.o + doc['owner'] = layer.owner.username if layer.owner else None +# doc['thumb'] = layer.get_thumbnail_url() + doc['last_modified'] = extension.date_fmt(layer.date) + doc['id'] = layer.id + doc['_type'] = 'layer' +# doc['owsUrl'] = layer.get_virtual_wms_url() + doc['topic'] = layer.topic_category + doc['name'] = layer.typename + doc['abstract'] = defaultfilters.linebreaks(layer.abstract) + doc['storeType'] = layer.storeType + doc['_display_type'] = extension.LAYER_DISPLAY + if 'bbox' not in exclude: + doc['bbox'] = _bbox(layer) + doc['keywords'] = layer.keyword_list() + doc['title'] = layer.title + doc['detail'] = layer.get_absolute_url() + #if 'download_links' not in exclude: + # links = layer.download_links() + # for i,e in enumerate(links): + # links[i] = [ unicode(l) for l in e] + # doc['download_links'] = links + owner = layer.owner + if owner: + doc['owner_detail'] = layer.owner.get_absolute_url() +# doc['owner_detail'] = reverse('about_storyteller', args=(layer.owner.username,)) + return doc + + +class OwnerNormalizer(Normalizer): + def title(self): + return self.o.user.get_full_name() or self.o.user.username + def last_modified(self): + try: + return self.o.user.date_joined + except ObjectDoesNotExist: + return None + def populate(self, doc, exclude): + contact = self.o + user = contact.user + try: + doc['thumb'] = user.avatar_set.all()[0].avatar_url(80) + except IndexError: + doc['thumb'] = _default_avatar_url + doc['id'] = user.username + doc['title'] = user.get_full_name() or user.username + doc['organization'] = contact.organization + doc['abstract'] = contact.profile + modified = self.last_modified() + doc['last_modified'] = extension.date_fmt(modified) if modified else '' + doc['detail'] = contact.get_absolute_url() + doc['layer_cnt'] = Layer.objects.filter(owner = user).count() + doc['map_cnt'] = Map.objects.filter(owner = user).count() + doc['_type'] = 'owner' + doc['_display_type'] = extension.USER_DISPLAY + return doc \ No newline at end of file diff --git a/geonode/search/backends/silage/populate_search_test_data.py b/geonode/search/backends/silage/populate_search_test_data.py new file mode 100644 index 00000000000..31defe660ed --- /dev/null +++ b/geonode/search/backends/silage/populate_search_test_data.py @@ -0,0 +1,129 @@ +from datetime import datetime +from datetime import timedelta +from django.core.serializers import serialize +from django.contrib.auth.models import User +from geonode.layers.models import Layer +from geonode.maps.models import Map +from geonode.people.models import Contact +from itertools import cycle +from taggit.models import Tag +from taggit.models import TaggedItem +from uuid import uuid4 +import os.path + + +# This is used to populate the database with the search fixture data. This is +# primarly used as a first step to generate the json data for the fixture using +# django's dumpdata + + +map_data = [ + ('lorem ipsum', 'common lorem ipsum', ('populartag',)), + ('ipsum lorem', 'common ipsum lorem', ('populartag', 'maptagunique')), + ('lorem1 ipsum1', 'common abstract1', ('populartag',)), + ('ipsum foo', 'common bar lorem', ('populartag',)), + ('map one', 'common this is a unique thing', ('populartag',)), + ('quux', 'common double thing', ('populartag',)), + ('morx', 'common thing double', ('populartag',)), + ('titledupe something else ', 'whatever common', ('populartag',)), + ('something titledupe else ', 'bar common', ('populartag',)), + ] + +user_data = [ + ('user1', 'pass', 'uniquefirst', 'foo'), + ('user2', 'pass', 'foo', 'uniquelast'), + ('unique_username', 'pass', 'foo', 'uniquelast'), + ('jblaze', 'pass', 'johnny', 'blaze'), + ('foo', 'pass', 'bar', 'baz'), + ] + +people_data = [ + ('this contains all my interesting profile information',), + ('some other information goes here',), + ] + +layer_data = [ + ('layer1', 'abstract1', 'layer1', 'geonode:layer1', [-180, 180, -90, 90], '19850101', ('populartag','here')), + ('layer2', 'abstract2', 'layer2', 'geonode:layer2', [-180, 180, -90, 90], '19800501', ('populartag',)), + ('uniquetitle', 'something here', 'mylayer', 'geonode:mylayer', [-180, 180, -90, 90], '19901001', ('populartag',)), + ('common blar', 'lorem ipsum', 'foo', 'geonode:foo', [-180, 180, -90, 90], '19000603', ('populartag', 'layertagunique')), + ('common double it', 'whatever', 'whatever', 'geonode:whatever', [0, 1, 0, 1], '50001101', ('populartag',)), + ('common double time', 'else', 'fooey', 'geonode:fooey', [0, 5, 0, 5], '00010101', ('populartag',)), + ('common bar', 'uniqueabstract', 'quux', 'geonode:quux', [0, 10, 0, 10], '19501209', ('populartag',)), + ('common morx', 'lorem ipsum', 'fleem', 'geonode:fleem', [0, 50, 0, 50], '19630829', ('populartag',)), + ] + + +def create_models(): + users = [] + for ud, pd in zip(user_data, cycle(people_data)): + user_name, password, first_name, last_name = ud + profile = pd[0] + u = User.objects.create_user(user_name) + u.first_name = first_name + u.last_name = last_name + u.save() + contact = Contact.objects.get(user=u) + contact.profile = profile + contact.save() + users.append(u) + + for md, user in zip(map_data, cycle(users)): + title, abstract, kws = md + m = Map(title=title, + abstract=abstract, + zoom=4, + projection='EPSG:4326', + center_x=42, + center_y=-73, + owner=user, + ) + m.save() + for kw in kws: + m.keywords.add(kw) + m.save() + + for ld, owner, storeType in zip(layer_data, cycle(users), cycle(('raster','vector'))): + title, abstract, name, typename, (bbox_x0, bbox_x1, bbox_y0, bbox_y1), dt, kws = ld + year, month, day = map(int, (dt[:4], dt[4:6], dt[6:])) + start = datetime(year, month, day) + end = start + timedelta(days=365) + l = Layer(title=title, + abstract=abstract, + name=name, + typename=typename, + bbox_x0=bbox_x0, + bbox_x1=bbox_x1, + bbox_y0=bbox_y0, + bbox_y1=bbox_y1, + uuid=str(uuid4()), + owner=owner, + temporal_extent_start=start, + temporal_extent_end=end, + storeType=storeType + ) + l.save() + for kw in kws: + l.keywords.add(kw) + l.save() + + +def dump_models(path=None): + result = serialize("json", sum([list(x) for x in + [User.objects.all(), + Contact.objects.all(), + Layer.objects.all(), + Map.objects.all(), + Tag.objects.all(), + TaggedItem.objects.all(), + ]], []), indent=2, use_natural_keys=True) + if path is None: + parent, _ = os.path.split(__file__) + path = os.path.join(parent, 'fixtures', 'silage_testdata.json') + with open(path, 'w') as f: + f.write(result) + + +if __name__ == '__main__': + create_models() + dump_models() diff --git a/geonode/search/backends/silage/query.py b/geonode/search/backends/silage/query.py new file mode 100644 index 00000000000..ffc2de48a4e --- /dev/null +++ b/geonode/search/backends/silage/query.py @@ -0,0 +1,197 @@ +######################################################################### +# +# Copyright (C) 2012 OpenPlans +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from geonode.search.backends.silage.util import resolve_extension +from geonode.utils import _split_query + +from django.conf import settings + +from datetime import date +from datetime import timedelta +import operator + +DEFAULT_MAPS_SEARCH_BATCH_SIZE = 10 + +_SEARCH_PARAMS = [ + 'type', + 'kw', + 'owner', + 'extent', + 'added', + 'period', + 'start', + 'end', + 'exclude', + 'cache'] + +# settings API +_search_config = getattr(settings,'SIMPLE_SEARCH_SETTINGS', {}) +_SEARCH_PARAMS.extend(_search_config.get('extra_query',[])) +_extra_context = resolve_extension('extra_context') +# end settings API + + +class BadQuery(Exception): + pass + + +class Query(object): + # while these are all class attributes, they will be overwritten + + # search params + query = None + 'search terms, query, keyword(s)' + + split_query = None + 'the query, but split into pieces' + + period = None + 'tuple of start/end date' + extent = None + + params = None + 'dict of specific field queries, use the fields for easier access' + + # sorting + sort = None + 'relevance, alpha, rating, created, updated' + + order = None + + # other + user = None + cache = True + + # paging + start = None + limit = None + + def __init__(self, query, start=0, limit=DEFAULT_MAPS_SEARCH_BATCH_SIZE, + sort_field='last_modified', sort_asc=False, filters=None, + user=None, cache=True): + self.query = query + self.split_query = _split_query(query) + self.start = start + self.limit = limit + self.sort = sort_field + self.order = sort_asc + self.params = filters or {} + self.user = user + self.cache = cache + + self.type = filters.get('type') + self.owner = filters.get('owner') + self.kw = filters.get('kw') + if self.kw: + self.kw = tuple(self.kw.split(',')) + + val = filters['period'] + self.period = tuple(val.split(',')) if val else None + + start = filters['start'] + end = filters['end'] + if start or end: + if self.period: + raise BadQuery('period and start/end both provided') + self.period = (start, end) + + val = filters['extent'] + if val: + try: + err = BadQuery('extent filter must contain x0,x1,y0,y1 comma separated') + parts = val.split(',') + if len(parts) != 4: + raise err + self.extent = map(float, parts) + except: + raise err + + val = filters['added'] + self.added = parse_by_added(val) if val else None + + + def cache_key(self): + '''the cache key is based on filters, the user and the text query''' + fhash = reduce(operator.xor, map(hash, self.params.items())) + return str(fhash ^ hash(self.user.username if self.user else 31) ^ hash(self.query)) + + + def get_query_response(self): + '''return a dict containing any non-null parameters used in the search''' + q = dict([ kv for kv in self.params.items() if kv[1] ]) + if self.query: + q['query'] = self.query + return q + + +def parse_by_added(spec): + if spec == 'today': + td = timedelta(days=1) + elif spec == 'week': + td = timedelta(days=7) + elif spec == 'month': + td = timedelta(days=30) + else: + raise BadQuery('valid added filter values are: today,week,month') + return date.today() - td + + +def query_from_request(request, extra): + params = dict(request.REQUEST) + params.update(extra) + + query = params.get('q', '') + try: + start = int(params.get('startIndex', 0)) + except ValueError: + raise BadQuery('startIndex must be valid number') + try: + limit = int(params.get('limit', DEFAULT_MAPS_SEARCH_BATCH_SIZE)) + except ValueError: + raise BadQuery('limit must be valid number') + + # handle old search link parameters + if 'sort' in params and 'dir' in params: + sort_field = params['sort'] + sort_asc = params['dir'] == 'ASC' + else: + sorts = { + 'newest' : ('last_modified',False), + 'oldest' : ('last_modified',True), + 'alphaaz' : ('title',True), + 'alphaza' : ('title',False), + 'popularity' : ('rank',False), + 'rel' : ('relevance',False) + } + try: + sort_field, sort_asc = sorts[params.get('sort','newest')] + except KeyError: + raise BadQuery('valid sorting values are: %s' % sorts.keys()) + + filters = dict([(k,params.get(k,None) or None) for k in _SEARCH_PARAMS]) + + aliases = dict(bbox='extent') + for k,v in aliases.items(): + if k in params: filters[v] = params[k] + + cache = bool(params.get('cache', True)) + return Query(query, start=start, limit=limit, sort_field=sort_field, + sort_asc=sort_asc, filters=filters, cache=cache, user=request.user) + + \ No newline at end of file diff --git a/geonode/search/backends/silage/search.py b/geonode/search/backends/silage/search.py new file mode 100644 index 00000000000..5f1c877b463 --- /dev/null +++ b/geonode/search/backends/silage/search.py @@ -0,0 +1,288 @@ +######################################################################### +# +# Copyright (C) 2012 OpenPlans +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.contrib.auth.models import Permission +from django.contrib.auth.models import User +from django.contrib.contenttypes.models import ContentType +from django.db import backend +from django.db.models import Q + +from geonode.security.models import UserObjectRoleMapping +from geonode.security.models import GenericObjectRoleMapping +from geonode.security.models import ANONYMOUS_USERS +from geonode.security.models import AUTHENTICATED_USERS +from geonode.maps.models import Layer +from geonode.maps.models import Map +from geonode.maps.models import MapLayer +from geonode.people.models import Contact + +from geonode.search.backends.silage import extension +from geonode.search.backends.silage.models import filter_by_period +from geonode.search.backends.silage.models import filter_by_extent +from geonode.search.backends.silage.models import using_geodjango + +import operator + + +def _rank_rules(model, *rules): + # prefix field names with model's db table to avoid ambiguity + return [('"%s"."%s"' % (model._meta.db_table, r[0]), r[1], r[2]) + for r in rules] + + +def _filter_results(l): + '''If the layer name doesn't match any of the patterns, it shows in the results''' + return not any(p.search(l['name']) for p in extension.exclude_regex) + + +def _filter_security(q, user, model, permission): + '''apply filters to the query that remove those model objects that are + not viewable by the given user based on row-level permissions''' + # superusers see everything + if user and user.is_superuser: return q + + # resolve the model permission + ct = ContentType.objects.get_for_model(model) + p = Permission.objects.get(content_type=ct, codename=permission) + + # apply generic role filters + generic_roles = [ANONYMOUS_USERS] + if user and not user.is_anonymous(): + generic_roles.append(AUTHENTICATED_USERS) + grm = GenericObjectRoleMapping.objects.filter(object_ct=ct, role__permissions__in=[p], subject__in=generic_roles).values('object_id') + q = q.filter(id__in=grm) + + # apply specific user filters + if user and not user.is_anonymous(): + urm = UserObjectRoleMapping.objects.filter(object_ct=ct, role__permissions__in=[p], user=user).values('object_id') + q = q | q.filter(id__in=urm) + # if the user is the owner, make sure these are included + q = q | getattr(model, 'objects').filter(owner=user) + + return q + +def _add_relevance(q, query, rank_rules): + # for unittests, it doesn't make sense to test this as it's postgres + # specific SQL - instead test/verify directly using a query and getting SQL + if 'sqlite' in backend.__name__: return q + +def _add_relevance(query, rank_rules): + eq = """CASE WHEN %s = '%s' THEN %s ELSE 0 END""" + frag = """CASE WHEN position(lower('%s') in lower(%s)) >= 1 THEN %s ELSE 0 END""" + + preds = [] + + preds.extend( [ eq % (r[0],query.query,r[1]) for r in rank_rules] ) + preds.extend( [ frag % (query.query,r[0],r[2]) for r in rank_rules] ) + + words = query.split_query + if len(words) > 1: + for w in words: + preds.extend( [ frag % (w,r[0],r[2] / 2) for r in rank_rules] ) + + sql = " + ".join(preds) + return sql + + +def _safely_add_relevance(q, query, rank_rules): + # for unittests, it doesn't make sense to test this as it's postgres + # specific SQL - instead test/verify directly using a query and getting SQL + if 'sqlite' in backend.__name__: return q + + sql = _add_relevance(query, rank_rules) + # ugh - work around bug + q = q.defer(None) + return q.extra(select={'relevance':sql}) + + +def _build_map_layer_text_query(q, query, query_keywords=False): + '''Build an OR query on title and abstract from provided search text. + if query_keywords is provided, include a query on the keywords attribute if + specified. + + return a Q object + ''' + # title or abstract contains entire phrase + subquery = [Q(title__icontains=query.query),Q(abstract__icontains=query.query)] + # tile or abstract contains pieces of entire phrase + if len(query.split_query) > 1: + subquery.extend([Q(title__icontains=kw) for kw in query.split_query]) + subquery.extend([Q(abstract__icontains=kw) for kw in query.split_query]) + # or keywords match any pieces of entire phrase + if query_keywords and query.split_query: + subquery.append(_build_kw_only_query(query.split_query)) + # if any OR phrases exists, build them + if subquery: + q = q.filter(reduce( operator.or_, subquery)) + return q + + +def _build_kw_only_query(keywords): + return reduce(operator.or_, [Q(keywords__name__contains=kw) for kw in keywords]) + + +def _get_owner_results(query): + # make sure all contacts have a user attached + q = extension.owner_query(query) + + if q is None: return None + + if query.kw: + # hard to handle - not supporting at the moment + return + + if query.owner: + q = q.filter(user__username__icontains = query.owner) + + if query.extent: + q = filter_by_extent(Map, q, query.extent, True) | \ + filter_by_extent(Layer, q, query.extent, True) + + if query.period: + q = filter_by_period(Map, q, *query.period, user=True) | \ + filter_by_period(Layer, q, *query.period, user=True) + + if query.added: + q = q.filter(user__date_joined__gt = query.added) + + if query.query: + qs = Q(user__username__icontains=query.query) | \ + Q(user__first_name__icontains=query.query) | \ + Q(user__last_name__icontains=query.query) + for field in extension.owner_query_fields: + qs = qs | Q(**{'%s__icontains' % field: query.query}) + q = q.filter(qs) + + rules = _rank_rules(User,['username', 10, 5]) + \ + _rank_rules(Contact,['organization', 5, 2]) + added = extension.owner_rank_rules() + if added: + rules = rules + _rank_rules(*added) + q = _safely_add_relevance(q, query, rules) + + return q.distinct() + + +def _get_map_results(query): + q = extension.map_query(query) + + q = _filter_security(q, query.user, Map, 'view_map') + + if query.owner: + q = q.filter(owner__username=query.owner) + + if query.extent: + q = filter_by_extent(Map, q, query.extent) + + if query.added: + q = q.filter(last_modified__gte=query.added) + + if query.period: + q = filter_by_period(Map, q, *query.period) + + if query.kw: + # this is a somewhat nested query but it performs way faster than + # other approaches + layers_with_kw = Layer.objects.filter(_build_kw_only_query(query.kw)).values('typename') + map_layers_with = MapLayer.objects.filter(name__in=layers_with_kw).values('map') + q = q.filter(id__in=map_layers_with) + + if query.query: + q = _build_map_layer_text_query(q, query, query_keywords=True) + rules = _rank_rules(Map, + ['title',10, 5], + ['abstract',5, 2], + ) + q = _safely_add_relevance(q, query, rules) + + return q.distinct() + + +def _get_layer_results(query): + + q = extension.layer_query(query) + + q = _filter_security(q, query.user, Layer, 'view_layer') + + if extension.exclude_patterns: + name_filter = reduce(operator.or_,[ Q(name__regex=f) for f in extension.exclude_patterns]) + q = q.exclude(name_filter) + + if query.kw: + q = q.filter(_build_kw_only_query(query.kw)) + + if query.owner: + q = q.filter(owner__username=query.owner) + + if query.type and query.type != 'layer': + q = q.filter(storeType = query.type) + + if query.extent: + q = filter_by_extent(Layer, q, query.extent) + + if query.added: + q = q.filter(date__gte=query.added) + + if query.period: + q = filter_by_period(Layer, q, *query.period) + + # this is a special optimization for prefetching results when requesting + # all records via search + # keywords and thumbnails cannot be prefetched at the moment due to + # the way the contenttypes are implemented + if query.limit == 0 and using_geodjango: + q = q.defer(None).prefetch_related("owner","spatial_temporal_index") + + if query.query: + q = _build_map_layer_text_query(q, query, query_keywords=True) |\ + q.filter(name__icontains=query.query) # map doesn't have name + rules = _rank_rules(Layer, + ['name',10, 1], + ['title',10, 5], + ['abstract',5, 2], + ) + q = _safely_add_relevance(q, query, rules) + + return q.distinct() + + +def combined_search_results(query): + facets = dict([ (k,0) for k in ('map', 'layer', 'vector', 'raster', 'user')]) + results = {'facets' : facets} + + bytype = query.type + + if bytype is None or bytype == u'map': + q = _get_map_results(query) + facets['map'] = q.count() + results['maps'] = q + + if bytype is None or bytype in (u'layer', u'raster', u'vector'): + q = _get_layer_results(query) + facets['layer'] = q.count() + facets['raster'] = q.filter(storeType='raster').count() + facets['vector'] = q.filter(storeType='vector').count() + results['layers'] = q + + if bytype is None or bytype == u'owner': + q = _get_owner_results(query) + facets['user'] = q.count() + results['owners'] = q + + return results diff --git a/geonode/search/backends/silage/tests.py b/geonode/search/backends/silage/tests.py new file mode 100644 index 00000000000..6882da0d991 --- /dev/null +++ b/geonode/search/backends/silage/tests.py @@ -0,0 +1,305 @@ +######################################################################### +# +# Copyright (C) 2012 OpenPlans +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.contrib.auth.models import User +from django.contrib.contenttypes.models import ContentType +from django.test.client import Client +from django.test import TestCase +from geonode.security.models import AUTHENTICATED_USERS +from geonode.security.models import ANONYMOUS_USERS +from geonode.layers.models import Layer +from geonode.maps.models import Map +from geonode.people.models import Contact +from geonode.search.backends.silage import search +from geonode.search.backends.silage import util +from geonode.search.backends.silage.query import query_from_request +from agon_ratings.models import OverallRating +import json +import logging + +logging.getLogger('south').setLevel(logging.INFO) +logging.getLogger('geonode.search.backends.silage.views').setLevel(logging.DEBUG) + +# quack +MockRequest = lambda **kw: type('xyz',(object,),{'REQUEST':kw,'user':None}) + +def all_public(): + '''ensure all layers and maps are publicly viewable''' + for l in Layer.objects.all(): + l.set_default_permissions() + for m in Map.objects.all(): + m.set_default_permissions() + +class SilageTest(TestCase): + + c = Client() + + fixtures = ['initial_data.json', 'silage_testdata.json'] + + @classmethod + def setUpClass(cls): + "Hook method for setting up class fixture before running tests in the class." + from django.core.cache import cache + cache.clear() + SilageTest('_fixture_setup')._fixture_setup(True) + all_public() + + @classmethod + def tearDownClass(cls): + "Hook method for deconstructing the class fixture after running all tests in the class." + SilageTest('_fixture_teardown')._fixture_teardown(True) + logging.getLogger('south').setLevel(logging.DEBUG) + + def _fixture_setup(self, a=False): + if a: + super(SilageTest, self)._fixture_setup() + + def _fixture_teardown(self, a=False): + if a: + super(SilageTest, self)._fixture_teardown() + + def request(self, query=None, **options): + query_dict = dict(q=query) if query else {} + get_params = dict(query_dict, **options) + return self.c.get('/search/api', get_params) + + def assert_results_contain_title(self, jsonvalue, title, _type=None): + matcher = (lambda doc: doc['title'] == title if _type is None else + lambda doc: doc['title'] == title and doc['_type'] == _type) + matches = filter(matcher, jsonvalue['results']) + self.assertTrue(matches, "No results match %s" % title) + + def search_assert(self, response, **options): + jsonvalue = json.loads(response.content) + + facets = jsonvalue['facets'] + if 'layer' in facets: + self.assertEquals(facets['raster'] + facets['vector'], facets['layer']) + +# import pprint; pprint.pprint(jsonvalue) + self.assertFalse(jsonvalue.get('errors')) + self.assertTrue(jsonvalue.get('success')) + + contains_maptitle = options.pop('contains_maptitle', None) + if contains_maptitle: + self.assert_results_contain_title(jsonvalue, contains_maptitle, 'map') + + contains_layertitle = options.pop('contains_layertitle', None) + if contains_layertitle: + self.assert_results_contain_title(jsonvalue, contains_layertitle, 'layer') + + contains_username = options.pop('contains_username', None) + if contains_username: + self.assert_results_contain_title(jsonvalue, contains_username, 'owner') + + n_results = options.pop('n_results', None) + if n_results: + self.assertEquals(n_results, len(jsonvalue['results'])) + + n_total = options.pop('n_total', None) + if n_total: + self.assertEquals(n_total, jsonvalue['total']) + + first_title = options.pop('first_title', None) + if first_title: + self.assertTrue(len(jsonvalue['results']) > 0, 'No results found') + doc = jsonvalue['results'][0] + self.assertEquals(first_title, doc['title']) + + sorted_by = options.pop('sorted_by', None) + if sorted_by: + reversed = sorted_by[0] == '-' + sorted_by = sorted_by.replace('-','') + sorted_fields = [ jv[sorted_by] for jv in jsonvalue['results'] ] + expected = list(sorted_fields) + expected.sort(reverse = reversed) + self.assertEquals(sorted_fields, expected) + + + def test_limit(self): + self.search_assert(self.request(limit=1), n_results=1) + + def test_query_map_title(self): + self.search_assert(self.request('unique'), contains_maptitle='map one') + + def test_query_layer_title(self): + self.search_assert(self.request('uniquetitle'), + contains_layerid='uniquetitle') + + def test_username(self): + self.search_assert(self.request('jblaze'), contains_username='jblaze') + + def test_profile(self): + self.search_assert(self.request("some other information"), + contains_username='jblaze') + + def test_text_across_types(self): + self.search_assert(self.request('foo'), n_results=7, n_total=7) + self.search_assert(self.request('common'), n_results=10, n_total=14) + + def test_pagination(self): + self.search_assert(self.request('common', startIndex=0), n_results=10, n_total=14) + self.search_assert(self.request('common', startIndex=10), n_results=4, n_total=14) + + def test_bbox_query(self): + # @todo since maps and users are excluded at the moment, this will have + # to be revisited + self.search_assert(self.request(extent='-180,180,-90,90'), n_results=8) + self.search_assert(self.request(extent='0,10,0,10'), n_results=3) + self.search_assert(self.request(extent='0,1,0,1'), n_results=1) + + def test_date_query(self): + self.search_assert(self.request(period='1980-01-01T00:00:00Z,1995-01-01T00:00:00Z'), + n_results=3) + self.search_assert(self.request(period=',1995-01-01T00:00:00Z'), + n_results=7) + self.search_assert(self.request(period='1980-01-01T00:00:00Z,'), + n_results=4) + + def test_errors(self): + self.assert_error(self.request(sort='foo'), + "valid sorting values are: ['alphaaz', 'newest', 'popularity', 'alphaza', 'rel', 'oldest']") + self.assert_error(self.request(extent='1,2,3'), + 'extent filter must contain x0,x1,y0,y1 comma separated') + self.assert_error(self.request(extent='a,b,c,d'), + 'extent filter must contain x0,x1,y0,y1 comma separated') + self.assert_error(self.request(startIndex='x'), + 'startIndex must be valid number') + self.assert_error(self.request(limit='x'), + 'limit must be valid number') + self.assert_error(self.request(added='x'), + 'valid added filter values are: today,week,month') + + def assert_error(self, resp, msg): + obj = json.loads(resp.content) + self.assertTrue(obj['success'] == False) + self.assertEquals(msg, obj['errors'][0]) + + def test_sort(self): + self.search_assert(self.request('foo', sort='newest'), + first_title='common double time', sorted_by='-last_modified') + self.search_assert(self.request('foo', sort='oldest'), + first_title='uniquefirst foo', sorted_by='last_modified') + self.search_assert(self.request('foo', sort='alphaaz'), + first_title='bar baz', sorted_by='title') + self.search_assert(self.request('foo', sort='alphaza'), + first_title='uniquefirst foo', sorted_by='-title') + + # apply some ratings + ct = ContentType.objects.get_for_model(Layer) + for l in Layer.objects.all(): + OverallRating.objects.create(content_type=ct, object_id=l.pk, rating=l.pk, category=2) + ct = ContentType.objects.get_for_model(Map) + for l in Map.objects.all(): + OverallRating.objects.create(content_type=ct, object_id=l.pk, rating=l.pk, category=1) + # clear any cached ratings + from django.core.cache import cache + cache.clear() + self.search_assert(self.request('foo', sort='popularity'), + first_title='common double time', sorted_by='-rating') + + def test_keywords(self): + # this tests the matching of the general query to keywords + self.search_assert(self.request('populartag'), n_results=10, n_total=17) + self.search_assert(self.request('maptagunique'), n_results=1, n_total=1) + self.search_assert(self.request('layertagunique'), n_results=1, n_total=1) + # verify little chunks must entirely match keywords + # po ma la are the prefixes to the former keywords :) + self.search_assert(self.request('po ma la'), n_results=0, n_total=0) + + def test_type_query(self): + self.search_assert(self.request('common', type='map'), n_results=9, n_total=9) + self.search_assert(self.request('common', type='layer'), n_results=5, n_total=5) + self.search_assert(self.request('foo', type='owner'), n_results=4, n_total=4) + # there are 8 total layers, half vector, half raster + self.search_assert(self.request('', type='raster'), n_results=4, n_total=4) + self.search_assert(self.request('', type='vector'), n_results=4, n_total=4) + + def test_kw_query(self): + # a kw-only query should filter out those not matching the keyword + self.search_assert(self.request('', kw='here', type='layer'), n_results=1, n_total=1) + # no matches + self.search_assert(self.request('', kw='foobar', type='layer'), n_results=0, n_total=0) + + def test_author_endpoint(self): + resp = self.c.get('/search/api/authors') + jsobj = json.loads(resp.content) + self.assertEquals(6, jsobj['total']) + + def test_search_page(self): + resp = self.c.get('/search/') + self.assertEquals(200, resp.status_code) + + def test_util(self): + jdate = util.iso_str_to_jdate('-5000-01-01T12:00:00Z') + self.assertEquals(jdate, -105192) + roundtripped = util.jdate_to_approx_iso_str(jdate) + self.assertEquals(roundtripped, '-4999-01-03') + + def test_security_trimming(self): + try: + self.run_security_trimming() + finally: + all_public() + + def run_security_trimming(self): + # remove permissions on all jblaze layers + jblaze_layers = Layer.objects.filter(owner__username='jblaze') + hiding = jblaze_layers.count() + for l in jblaze_layers: + l.set_gen_level(ANONYMOUS_USERS, l.LEVEL_NONE) + l.set_gen_level(AUTHENTICATED_USERS, l.LEVEL_NONE) + + # a (anonymous) layer query should exclude the number of hiding layers + self.search_assert(self.request(type='layer'), n_results=8 - hiding, n_total=8 - hiding) + + # admin sees all + self.assertTrue(self.c.login(username='admin', password='admin')) + self.search_assert(self.request(type='layer'), n_results=8, n_total=8) + self.c.logout() + + # a logged in jblaze will see his, too + jblaze = User.objects.get(username='jblaze') + jblaze.set_password('passwd') + jblaze.save() + self.assertTrue(self.c.login(username='jblaze', password='passwd')) + self.search_assert(self.request(type='layer'), n_results=8, n_total=8) + self.c.logout() + + def test_relevance(self): + query = query_from_request(MockRequest(q='foo'), {}) + + def assert_rules(rules): + rank_rules = [] + for model, model_rules in rules: + rank_rules.extend(search._rank_rules(model, *model_rules)) + + sql = search._add_relevance(query, rank_rules) + + for _, model_rules in rules: + for attr, rank1, rank2 in model_rules: + self.assertTrue(('THEN %d ELSE 0' % rank1) in sql) + self.assertTrue(('THEN %d ELSE 0' % rank2) in sql) + self.assertTrue(attr in sql) + + assert_rules([(Map, [('title', 10, 5), ('abstract', 5, 2)])]) + assert_rules([(Layer, + [('name', 10, 1), ('title', 10, 5), ('abstract', 5, 2)])]) + assert_rules([(User, [('username', 10, 5)]), + (Contact, [('organization', 5, 2)])]) diff --git a/geonode/search/backends/silage/urls.py b/geonode/search/backends/silage/urls.py new file mode 100644 index 00000000000..5ba65e522e2 --- /dev/null +++ b/geonode/search/backends/silage/urls.py @@ -0,0 +1,28 @@ +######################################################################### +# +# Copyright (C) 2012 OpenPlans +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.conf.urls.defaults import * + +urlpatterns = patterns('geonode.search.backends.silage.views', + url(r'^$', 'search_page', name='search'), + url(r'^api$', 'search_api', name='search_api'), + url(r'^api/data$', 'search_api', kwargs={'type':'layer'}, name='layer_search_api'), + url(r'^api/maps$', 'search_api', kwargs={'type':'map'}, name='maps_search_api'), + url(r'^api/authors$', 'author_list', name='search_api_author_list'), +) \ No newline at end of file diff --git a/geonode/search/backends/silage/util.py b/geonode/search/backends/silage/util.py new file mode 100644 index 00000000000..e989ac1b266 --- /dev/null +++ b/geonode/search/backends/silage/util.py @@ -0,0 +1,185 @@ +######################################################################### +# +# Copyright (C) 2012 OpenPlans +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +import math +from django.conf import settings + +_search_config = getattr(settings,'SIMPLE_SEARCH_SETTINGS', {}) +_extension = _search_config.get('extension', None) +if _extension: + _extension = __import__(_extension,level=0,fromlist=['*']) + +iso_fmt = '%Y-%m-%dT%H:%M:%SZ' + +def resolve_extension(name): + if _extension is None: return None + return getattr(_extension,name,None) + +'''taken from http://oneau.wordpress.com/2010/06/20/julian-date-calculator/''' + +MJD0 = 2400000.5 # 1858 November 17, 00:00:00 hours + +def base60_to_decimal(xyz,delimiter=None): + """Decimal value from numbers in sexagesimal system. + + The input value can be either a floating point number or a string + such as "hh mm ss.ss" or "dd mm ss.ss". Delimiters other than " " + can be specified using the keyword ``delimiter``. + """ + + divisors = [1,60.0,3600.0] + xyzlist = str(xyz).split(delimiter) + sign = -1 if xyzlist[0].find("-") != -1 else 1 + + xyzlist = [abs(float(x)) for x in xyzlist] + decimal_value = 0 + + for i,j in zip(xyzlist,divisors): # if xyzlist has <3 values then + # divisors gets clipped. + decimal_value += i/j + + decimal_value = -decimal_value if sign == -1 else decimal_value + return decimal_value + +def decimal_to_base60(deci,precision=1e-8): + """Converts decimal number into sexagesimal number parts. + + ``deci`` is the decimal number to be converted. ``precision`` is how + close the multiple of 60 and 3600, for example minutes and seconds, + are to 60.0 before they are rounded to the higher quantity, for + + example hours and minutes. + """ + sign = "+" # simple putting sign back at end gives errors for small + # deg. This is because -00 is 00 and hence ``format``, + # that constructs the delimited string will not add '-' + + # sign. So, carry it as a character. + if deci < 0: + deci = abs(deci) + sign = "-" + + frac1, num = math.modf(deci) + num = int(num) # hours/degrees is integer valued but type is float + frac2, frac1 = math.modf(frac1*60.0) + frac1 = int(frac1) # minutes is integer valued but type is float + + frac2 *= 60.0 # number of seconds between 0 and 60 + + # Keep seconds and minutes in [0 - 60.0000) + if abs(frac2 - 60.0) < precision: + frac2 = 0.0 + + frac1 += 1 + if abs(frac1 - 60.0) < precision: + frac1 = 0.0 + + num += 1 + + return (sign,num,frac1,frac2) + +def julian_date(year,month,day,hour=0,minute=0,second=0): + """Given year, month, day, hour, minute and second return JD. + + ``year``, ``month``, ``day``, ``hour`` and ``minute`` are integers, + truncates fractional part; ``second`` is a floating point number. + For BC year: use -(year-1). Example: 1 BC = 0, 1000 BC = -999. + """ + MJD0 = 2400000.5 # 1858 November 17, 00:00:00 hours + + year, month, day, hour, minute =\ + int(year),int(month),int(day),int(hour),int(minute) + + if month <= 2: + month +=12 + + year -= 1 + + modf = math.modf + # Julian calendar on or before 1582 October 4 and Gregorian calendar + # afterwards. + + if ((10000L*year+100L*month+day) <= 15821004L): + b = -2 + int(modf((year+4716)/4)[1]) - 1179 + else: + b = int(modf(year/400)[1])-int(modf(year/100)[1])+\ + int(modf(year/4)[1]) + + mjdmidnight = 365L*year - 679004L + b + int(30.6001*(month+1)) + day + + fracofday = base60_to_decimal(\ + " ".join([str(hour),str(minute),str(second)])) / 24.0 + + return MJD0 + mjdmidnight + fracofday + +def caldate(mjd): + """Given mjd return calendar date. + + Retrns a tuple (year,month,day,hour,minute,second). The last is a + floating point number and others are integers. The precision in + seconds is about 1e-4. + + To convert jd to mjd use jd - 2400000.5. In this module 2400000.5 is + stored in MJD0. + + """ + MJD0 = 2400000.5 # 1858 November 17, 00:00:00 hours + + modf = math.modf + a = long(mjd+MJD0+0.5) + # Julian calendar on or before 1582 October 4 and Gregorian calendar + + # afterwards. + if a < 2299161: + b = 0 + c = a + 1524 + + else: + b = long((a-1867216.25)/36524.25) + c = a+ b - long(modf(b/4)[1]) + 1525 + + d = long((c-122.1)/365.25) + e = 365*d + long(modf(d/4)[1]) + f = long((c-e)/30.6001) + + day = c - e - int(30.6001*f) + month = f - 1 - 12*int(modf(f/14)[1]) + year = d - 4715 - int(modf((7+month)/10)[1]) + fracofday = mjd - math.floor(mjd) + hours = fracofday * 24.0 + + sign,hour,minute,second = decimal_to_base60(hours) + + return (year,month,day,int(sign+str(hour)),minute,second) + +def iso_str_to_jdate(iso_str): + ymd = iso_str.split('T')[0] + neg = 1 + if ymd[0] == '-': + ymd = ymd[1:] + neg = -1 + args = map(int,ymd.split('-')) + args = args + ([1] * (3 - len(args))) + args[0] *= neg + return int(julian_date(*args)) + +def jdate_to_approx_iso_str(jdate): + if jdate is None: return None + y,m,d,h,m,s = caldate(jdate - MJD0) + return '%.4d-%.2d-%.2d' % (y,m + 1,d) diff --git a/geonode/search/backends/silage/views.py b/geonode/search/backends/silage/views.py new file mode 100644 index 00000000000..b64fd7a5290 --- /dev/null +++ b/geonode/search/backends/silage/views.py @@ -0,0 +1,227 @@ +######################################################################### +# +# Copyright (C) 2012 OpenPlans +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +######################################################################### + +from django.db import connection +from django.http import HttpResponse +from django.shortcuts import render_to_response +from django.conf import settings +from django.template import RequestContext +from django.contrib.auth.models import User +from django.core.cache import cache + +from geonode.maps.views import default_map_config +from geonode.maps.models import Layer +from geonode.maps.models import Map +from geonode.people.models import Contact +from geonode.search.backends.silage.search import combined_search_results +from geonode.search.backends.silage.util import resolve_extension +from geonode.search.backends.silage.normalizers import apply_normalizers +from geonode.search.backends.silage.query import query_from_request +from geonode.search.backends.silage.query import BadQuery + +from datetime import datetime +from time import time +import json +import cPickle as pickle +import operator +import logging +import zlib + +logger = logging.getLogger(__name__) + +_extra_context = resolve_extension('extra_context') + +DEFAULT_MAPS_SEARCH_BATCH_SIZE = 10 + + +def _create_viewer_config(): + DEFAULT_MAP_CONFIG, DEFAULT_BASE_LAYERS = default_map_config() + _map = Map(projection="EPSG:900913", zoom = 1, center_x = 0, center_y = 0) + return json.dumps(_map.viewer_json(*DEFAULT_BASE_LAYERS)) +_viewer_config = _create_viewer_config() + + +def search_page(request, **kw): + params = {} + if kw: + params.update(kw) + + context = _get_search_context() + context['init_search'] = json.dumps(params) + + return render_to_response('silage/search.html', RequestContext(request, context)) + + +def _get_search_context(): + cache_key = 'simple_search_context' + context = cache.get(cache_key) + if context: return context + + counts = { + 'maps' : Map.objects.count(), + 'layers' : Layer.objects.count(), + 'vector' : Layer.objects.filter(storeType='dataStore').count(), + 'raster' : Layer.objects.filter(storeType='coverageStore').count(), + 'users' : Contact.objects.count() + } + topics = Layer.objects.all().values_list('topic_category',flat=True) + topic_cnts = {} + for t in topics: topic_cnts[t] = topic_cnts.get(t,0) + 1 + context = { + 'viewer_config': _viewer_config, + 'GOOGLE_API_KEY' : settings.GOOGLE_API_KEY, + "site" : settings.SITEURL, + 'counts' : counts, + 'users' : User.objects.all(), + 'topics' : topic_cnts, + 'keywords' : _get_all_keywords() + } + if _extra_context: + _extra_context(context) + cache.set(cache_key, context, 60) + + return context + + +def _get_all_keywords(): + allkw = {} + # @todo tagging added to maps and contacts, depending upon search type, + # need to get these... for now it doesn't matter (in mapstory) as + # only layers support keywords ATM. + for l in Layer.objects.all().select_related().only('keywords'): + kw = [ k.name for k in l.keywords.all() ] + for k in kw: + allkw[k] = allkw.get(k,0) + 1 + + return allkw + + +def search_api(request, **kwargs): + if request.method not in ('GET','POST'): + return HttpResponse(status=405) + debug = logger.isEnabledFor(logging.DEBUG) + if debug: + connection.queries = [] + ts = time() + try: + query = query_from_request(request, kwargs) + items, facets = _search(query) + ts1 = time() - ts + if debug: + ts = time() + results = _search_json(query, items, facets, ts1) + if debug: + ts2 = time() - ts + logger.debug('generated combined search results in %s, %s',ts1,ts2) + logger.debug('with %s db queries',len(connection.queries)) + return results + except Exception, ex: + if not isinstance(ex, BadQuery): + logger.exception("error during search") + return HttpResponse(json.dumps({ + 'success' : False, + 'errors' : [str(ex)] + }), status=400) + + +def _search_json(query, items, facets, time): + total = len(items) + + if query.limit > 0: + items = items[query.start:query.start + query.limit] + + # unique item id for ext store (this could be done client side) + iid = query.start + for r in items: + r.iid = iid + iid += 1 + + exclude = query.params.get('exclude') + exclude = set(exclude.split(',')) if exclude else () + items = map(lambda r: r.as_dict(exclude), items) + + results = { + '_time' : time, + 'results' : items, + 'total' : total, + 'success' : True, + 'query' : query.get_query_response(), + 'facets' : facets + } + return HttpResponse(json.dumps(results), mimetype="application/json") + + +def cache_key(query,filters): + return str(reduce(operator.xor,map(hash,filters.items())) ^ hash(query)) + + +def _search(query): + # to support super fast paging results, cache the intermediates + results = None + cache_time = 60 + if query.cache: + key = query.cache_key() + results = cache.get(key) + if results: + # put it back again - this basically extends the lease + cache.add(key, results, cache_time) + + if not results: + results = combined_search_results(query) + facets = results['facets'] + results = apply_normalizers(results) + if query.cache: + dumped = zlib.compress(pickle.dumps((results, facets))) + logger.debug("cached search results %s", len(dumped)) + cache.set(key, dumped, cache_time) + + else: + results, facets = pickle.loads(zlib.decompress(results)) + + # @todo - sorting should be done in the backend as it can optimize if + # the query is restricted to one model. has implications for caching... + if query.sort == 'title': + keyfunc = lambda r: r.title().lower() + elif query.sort == 'last_modified': + old = datetime(1,1,1) + keyfunc = lambda r: r.last_modified() or old + else: + keyfunc = lambda r: getattr(r, query.sort)() + results.sort(key=keyfunc, reverse=not query.order) + + return results, facets + + +def author_list(req): + q = User.objects.all() + + query = req.REQUEST.get('query',None) + start = int(req.REQUEST.get('start',0)) + limit = int(req.REQUEST.get('limit',20)) + + if query: + q = q.filter(username__icontains=query) + + vals = q.values_list('username',flat=True)[start:start+limit] + results = { + 'total' : q.count(), + 'names' : [ dict(name=v) for v in vals ] + } + return HttpResponse(json.dumps(results), mimetype="application/json") + \ No newline at end of file From 751d4d79f9b1727d852b769d43e0d7b50689f72a Mon Sep 17 00:00:00 2001 From: Ian Schneider Date: Wed, 17 Oct 2012 16:17:30 -0600 Subject: [PATCH 2/2] core adjustments for silage, mostly urls - in addition to url changes, adjust js template to use results instead of rows (as per search spec) - add map changed signal to allow hooking of indexer - add function to add bbox filter to a query - implement local_layers - remove obsolete integration test (covered in unit tests now) --- geonode/layers/models.py | 13 +++ .../layers/templates/layers/layer_search.html | 4 +- geonode/layers/tests.py | 2 +- geonode/layers/urls.py | 1 - geonode/maps/models.py | 10 +- geonode/maps/signals.py | 3 + geonode/maps/templates/maps/map_search.html | 4 +- geonode/maps/tests.py | 6 +- geonode/maps/urls.py | 1 - geonode/settings.py | 1 + geonode/smoke_tests.py | 2 +- geonode/tests/integration.py | 93 +------------------ geonode/urls.py | 3 + 13 files changed, 43 insertions(+), 100 deletions(-) create mode 100644 geonode/maps/signals.py diff --git a/geonode/layers/models.py b/geonode/layers/models.py index ed019b06821..4ce7694c3d4 100644 --- a/geonode/layers/models.py +++ b/geonode/layers/models.py @@ -271,6 +271,19 @@ def keyword_csv(self): class Meta: abstract = True + + +def add_bbox_query(q, bbox): + '''modify the queryset q to limit to the provided bbox + + bbox - 4 tuple of floats representing x0,x1,y0,y1 + returns the modified query + ''' + q = q.filter(bbox_x0__gte=bbox[0]) + q = q.filter(bbox_x1__lte=bbox[1]) + q = q.filter(bbox_y0__gte=bbox[2]) + return q.filter(bbox_y1__lte=bbox[3]) + class Layer(ResourceBase): """ diff --git a/geonode/layers/templates/layers/layer_search.html b/geonode/layers/templates/layers/layer_search.html index 34e6cb2a2b4..7f58e06a9b4 100644 --- a/geonode/layers/templates/layers/layer_search.html +++ b/geonode/layers/templates/layers/layer_search.html @@ -79,14 +79,14 @@

{% trans "Selected Data" %}

{% block extra_script %}