Merge pull request #6686 from IQSS/4964-harvesting-issues

4964 harvesting issues

Author: kcondon

src/main/java/edu/harvard/iq/dataverse/Dataset.java

@@ -32,6 +32,7 @@
 import javax.persistence.Temporal;
 import javax.persistence.TemporalType;
 import edu.harvard.iq.dataverse.util.BundleUtil;
+import edu.harvard.iq.dataverse.util.StringUtil;
 
 /**
  *
@@ -753,7 +754,32 @@ public String getRemoteArchiveURL() {
                     }
                 }
                 return this.getHarvestedFrom().getArchiveUrl();
+            } else if (HarvestingClient.HARVEST_STYLE_DEFAULT.equals(this.getHarvestedFrom().getHarvestStyle())) {
+                // This is a generic OAI archive. 
+                // The metadata we harvested for this dataset is most likely a 
+                // simple DC record that does not contain a URL pointing back at 
+                // the specific location on the source archive. But, it probably
+                // has a global identifier, a DOI or a Handle - so we should be 
+                // able to redirect to the proper global resolver. 
+                // But since this is a harvested dataset, we will assume that 
+                // there is a possibility tha this object does NOT have all the 
+                // valid persistent identifier components.
+                
+                if (StringUtil.nonEmpty(this.getProtocol()) 
+                        && StringUtil.nonEmpty(this.getAuthority())
+                        && StringUtil.nonEmpty(this.getIdentifier())) {
+                    return this.getPersistentURL();    
+                }
+                
+                // All we can do is redirect them to the top-level URL we have 
+                // on file for this remote archive:
+                return this.getHarvestedFrom().getArchiveUrl();
             } else {
+                // This is really not supposed to happen - this is a harvested
+                // dataset for which we don't have ANY information on the nature
+                // of the archive we got it from. So all we can do is redirect 
+                // the user to the top-level URL we have on file for this remote 
+                // archive:
                 return this.getHarvestedFrom().getArchiveUrl();
             }
         }

src/main/java/edu/harvard/iq/dataverse/HarvestingClientsPage.java

@@ -20,6 +20,7 @@
 import edu.harvard.iq.dataverse.util.BundleUtil;
 import edu.harvard.iq.dataverse.util.JsfHelper;
 import static edu.harvard.iq.dataverse.util.JsfHelper.JH;
+import edu.harvard.iq.dataverse.util.StringUtil;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -77,6 +78,7 @@ public class HarvestingClientsPage implements java.io.Serializable {
     private Dataverse dataverse;
     private Long dataverseId = null;
     private HarvestingClient selectedClient;
+    private boolean setListTruncated = false; 
 
     //private static final String solrDocIdentifierDataset = "dataset_";
 
@@ -206,6 +208,9 @@ public boolean isCreateStepFour() {
         return CreateStep.FOUR == this.createStep;
     }
 
+    public boolean isSetListTruncated() {
+        return setListTruncated;
+    }
 
     public void runHarvest(HarvestingClient harvestingClient) {
         try {
@@ -246,7 +251,7 @@ public void editClient(HarvestingClient harvestingClient) {
         // and if not, what do we do? 
         // alternatively, should we make these 2 fields not editable at all?
 
-        this.newOaiSet = !StringUtils.isEmpty(harvestingClient.getHarvestingSet()) ? harvestingClient.getHarvestingSet() : "none";
+        this.newOaiSet = !StringUtils.isEmpty(harvestingClient.getHarvestingSet()) ? harvestingClient.getHarvestingSet() : "";
         this.newMetadataFormat = harvestingClient.getMetadataPrefix();
         this.newHarvestingStyle = harvestingClient.getHarvestStyle();
 
@@ -492,6 +497,23 @@ public void validateMetadataFormat(FacesContext context, UIComponent toValidate,
         }
     }
 
+    public void validateRemoteArchiveStyle(FacesContext context, UIComponent toValidate, Object rawValue) {
+        String value = (String) rawValue;
+        UIInput input = (UIInput) toValidate;
+        input.setValid(true); // Optimistic approach
+        
+        // the only validation we want is to make sure the select one of the 
+        // values from the menu. 
+        if (context.getExternalContext().getRequestParameterMap().get("DO_VALIDATION") != null
+                && StringUtils.isEmpty(value)) {
+
+            input.setValid(false);
+            context.addMessage(toValidate.getClientId(),
+                    new FacesMessage(FacesMessage.SEVERITY_ERROR, "", BundleUtil.getStringFromBundle("harvestclients.newClientDialog.harvestingStyle.required")));
+
+        }
+    }
+    
     public boolean validateNickname() {
 
         if ( !StringUtils.isEmpty(getNewNickname()) ) {
@@ -559,10 +581,15 @@ public boolean validateServerUrlOAI() {
             }
             // And if that worked, the list of sets provided:
 
+            ArrayList<String> sets = null;
+            
+            // reset the sets menu:
+            setOaiSetsSelectItems(null);
+            setListTruncated = false;
+            
             if (success) {
                 try {
-                    List<String> sets = oaiHandler.runListSets();
-                    createOaiSetsSelectItems(sets);
+                    sets = oaiHandler.runListSets();
                 } catch (Exception ex) {
                     //success = false; 
                     // ok - we'll try and live without sets for now... 
@@ -576,6 +603,24 @@ public boolean validateServerUrlOAI() {
             }
 
             if (success) {
+                if (sets != null) {
+                    if (oaiHandler.isSetListTruncated()) {
+                        // If it was taking too long to retrieve the full list 
+                        // of sets (oai.datacite.org/oai - looking at you! -
+                        // and we had to truncate it:
+                        setListTruncated = true;
+
+                        // And if we are re-configuring an existing client, with 
+                        // a specific set in place - let's make sure it's on the pull down 
+                        // menu list; even if we have failed to retrieve it from the server. 
+                        if (StringUtil.nonEmpty(this.newOaiSet)) {
+                            if (!sets.contains(this.newOaiSet)) {
+                                sets.add(0, this.newOaiSet);
+                            }
+                        }
+                    }
+                    createOaiSetsSelectItems(sets);
+                }
                 return true;
             }
 
@@ -643,7 +688,8 @@ public void backToStepThree() {
     UIInput newClientNicknameInputField;
     UIInput newClientUrlInputField;
     UIInput hiddenInputField; 
-    /*UISelectOne*/ UIInput metadataFormatMenu; 
+    /*UISelectOne*/ UIInput metadataFormatMenu;
+    UIInput remoteArchiveStyleMenu;
     UIInput selectedDataverseMenu;
 
     private String newNickname = "";
@@ -674,7 +720,7 @@ public void initNewClient(ActionEvent ae) {
         this.initialSettingsValidated = false;
         this.newOaiSet = "";
         this.newMetadataFormat = "";
-        this.newHarvestingStyle = HarvestingClient.HARVEST_STYLE_DATAVERSE;
+        this.newHarvestingStyle = "";
 
         this.harvestTypeRadio = harvestTypeRadioOAI;
         this.harvestingScheduleRadio = harvestingScheduleRadioNone; 
@@ -840,6 +886,14 @@ public void setMetadataFormatMenu(UIInput metadataFormatMenu) {
         this.metadataFormatMenu = metadataFormatMenu;
     }
 
+    public UIInput getRemoteArchiveStyleMenu() {
+        return remoteArchiveStyleMenu;
+    }
+
+    public void setRemoteArchiveStyleMenu(UIInput remoteArchiveStyleMenu) {
+        this.remoteArchiveStyleMenu = remoteArchiveStyleMenu;
+    }
+    
     public UIInput getSelectedDataverseMenu() {
         return selectedDataverseMenu;
     }

src/main/java/edu/harvard/iq/dataverse/harvest/client/HarvestingClient.java

@@ -90,7 +90,7 @@ public void setId(Long id) {
     public static final String HARVEST_STYLE_DESCRIPTION_NESSTAR="Nesstar archive";
     public static final String HARVEST_STYLE_DESCRIPTION_ROPER="Roper Archive";
     public static final String HARVEST_STYLE_DESCRIPTION_HGL="HGL";
-    public static final String HARVEST_STYLE_DESCRIPTION_DEFAULT="Generic OAI resource (DC)";
+    public static final String HARVEST_STYLE_DESCRIPTION_DEFAULT="Generic OAI archive";
 
 
     public static final List<String> HARVEST_STYLE_LIST = Arrays.asList(HARVEST_STYLE_DATAVERSE, HARVEST_STYLE_VDC, HARVEST_STYLE_ICPSR, HARVEST_STYLE_NESSTAR, HARVEST_STYLE_ROPER, HARVEST_STYLE_HGL, HARVEST_STYLE_DEFAULT);

src/main/java/edu/harvard/iq/dataverse/harvest/client/oai/OaiHandler.java

@@ -82,6 +82,7 @@ public OaiHandler(HarvestingClient harvestingClient) throws OaiHandlerException
     private String metadataPrefix; // = harvestingClient.getMetadataPrefix();
     private String setName; 
     private Date   fromDate;
+    private Boolean setListTruncated = false;
 
     private ServiceProvider serviceProvider; 
 
@@ -123,6 +124,9 @@ public void setHarvestingClient(HarvestingClient harvestingClient) {
         this.harvestingClient = harvestingClient; 
     }
 
+    public boolean isSetListTruncated() {
+        return setListTruncated;
+    }
 
     private ServiceProvider getServiceProvider() throws OaiHandlerException {
         if (serviceProvider == null) {
@@ -141,12 +145,14 @@ private ServiceProvider getServiceProvider() throws OaiHandlerException {
         return serviceProvider;
     }
 
-    public List<String> runListSets() throws OaiHandlerException {
+    public ArrayList<String> runListSets() throws OaiHandlerException {
 
         ServiceProvider sp = getServiceProvider(); 
 
         Iterator<Set> setIter;
 
+        long startMilSec = new Date().getTime();
+        
         try {
             setIter = sp.listSets();
         } catch (NoSetHierarchyException nshe) {
@@ -155,9 +161,12 @@ public List<String> runListSets() throws OaiHandlerException {
             throw new OaiHandlerException("No valid response received from the OAI server.");
         }
 
-        List<String> sets = new ArrayList<>();
+        ArrayList<String> sets = new ArrayList<>();
 
+        int count = 0;
+        
         while ( setIter.hasNext()) {
+            count++;
             Set set = setIter.next();
             String setSpec = set.getSpec();
             /*
@@ -166,6 +175,15 @@ public List<String> runListSets() throws OaiHandlerException {
                 
             }
             */
+            
+            if (count >= 100) {
+                // Have we been waiting more than 30 seconds?
+                if (new Date().getTime() - startMilSec > 30000) {
+                    setListTruncated = true;
+                    break; 
+                }
+            }
+                         
             if (!StringUtils.isEmpty(setSpec)) {
                 sets.add(setSpec);
             }

src/main/java/propertyFiles/Bundle.properties

@@ -482,6 +482,7 @@ harvestclients.newClientDialog.oaiSets.tip=Harvesting sets offered by this OAI s
 harvestclients.newClientDialog.oaiSets.noset=None
 harvestclients.newClientDialog.oaiSets.helptext=Selecting "none" will harvest the default set, as defined by the server. Often this will be the entire body of content across all sub-sets.
 harvestclients.newClientDialog.oaiSets.helptext.noset=This OAI server does not support named sets. The entire body of content offered by the server will be harvested.
+harvestclients.newClientDialog.oaiSets.listTruncated=Please note that the remote server was taking too long to return the full list of available OAI sets, so the list was truncated. Please select a set from the current list (or select the "no set" option), and try again later, if you need to change it. 
 harvestclients.newClientDialog.oaiMetadataFormat=Metadata Format
 harvestclients.newClientDialog.oaiMetadataFormat.tip=Metadata formats offered by the remote server.
 harvestclients.newClientDialog.oaiMetadataFormat.required=Please select the metadata format to harvest from this archive.
@@ -503,6 +504,7 @@ harvestclients.newClientDialog.step4=Step 4 of 4 - Display
 harvestclients.newClientDialog.harvestingStyle=Archive Type
 harvestclients.newClientDialog.harvestingStyle.tip=Type of remote archive.
 harvestclients.newClientDialog.harvestingStyle.helptext=Select the archive type that best describes this remote server in order to properly apply formatting rules and styles to the harvested metadata as they are shown in the search results. Note that improperly selecting the type of the remote archive can result in incomplete entries in the search results, and a failure to redirect the user to the archival source of the data.
+harvestclients.newClientDialog.harvestingStyle.required=Please select one of the values from the menu.
 harvestclients.viewEditDialog.title=Edit Harvesting Client
 harvestclients.viewEditDialog.archiveUrl=Archive URL
 harvestclients.viewEditDialog.archiveUrl.tip=The URL of the archive that serves the data harvested by this client, which is used in search results for links to the original sources of the harvested content.

src/main/webapp/harvestclients.xhtml

@@ -320,13 +320,18 @@
                                             <f:selectItems value="#{harvestingClientsPage.oaiSetsSelectItems}" />
                                         </p:selectOneMenu>
                                         <p:message for="serverOaiSets"/>
-                                        <!-- a readonly label, in "edit existing client" mode: -->
+                                        <!-- a readonly label, in "edit existing client" mode: (No longer needed, since we now allow editing sets in existing clients! -->
                                         <p:inputText rendered="#{false}"
                                                      readonly="true"
                                                      styleClass="form-control"
                                                      value="#{harvestingClientsPage.newOaiSet}"/>
                                         <p class="help-block" jsf:rendered="#{harvestingClientsPage.createMode and !empty harvestingClientsPage.oaiSetsSelectItems}">#{bundle['harvestclients.newClientDialog.oaiSets.helptext']}</p>
                                         <p class="help-block" jsf:rendered="#{harvestingClientsPage.createMode and empty harvestingClientsPage.oaiSetsSelectItems}">#{bundle['harvestclients.newClientDialog.oaiSets.helptext.noset']}</p>
+                                        
+                                        <p class="help-block" jsf:rendered="#{harvestingClientsPage.setListTruncated}">
+                                            <span class="glyphicon glyphicon-warning-sign text-warning"/>
+                                            #{bundle['harvestclients.newClientDialog.oaiSets.listTruncated']}
+                                        </p>
                                     </div>
                                 </div>
 
@@ -339,7 +344,7 @@
                                               data-toggle="tooltip" data-placement="auto right" data-original-title="#{bundle['harvestclients.newClientDialog.oaiMetadataFormat.tip']}"></span>
                                     </label>
                                     <div class="col-sm-6">
-                                        <!-- select menu, shown in "create new client" mode, after the server has been contacted and we got the list of available metadata formats: -->
+                                        <!-- select menu; it's populated by contacting the remote server and obtaining the list of available metadata formats: -->
                                         <p:selectOneMenu id="oaiMetadataFormat"
                                                          binding="#{harvestingClientsPage.metadataFormatMenu}"
                                                          styleClass="form-control"
@@ -350,11 +355,11 @@
                                             <f:selectItems value="#{harvestingClientsPage.oaiMetadataFormatSelectItems}" />
                                         </p:selectOneMenu>
                                         <p:message for="oaiMetadataFormat"/>
-                                        <!-- a readonly label, in "edit existing client" mode: -->
-                                        <p:inputText rendered="#{harvestingClientsPage.editMode}"
-                                                     readonly="true"
+                                        <!-- a readonly label, in "edit existing client" mode: (No longer needed! - since we now allow formats to be edited) -->
+                                        <!-- p:inputText rendered="#{harvestingClientsPage.editMode}"
+                                                     readonly="false" 
                                                      styleClass="form-control"
-                                                     value="#{harvestingClientsPage.selectedClient.metadataPrefix}"/>
+                                                     value="#{harvestingClientsPage.selectedClient.metadataPrefix}"/ -->
                                     </div>
                                 </div>
 
@@ -487,8 +492,12 @@
                                     </label>
                                     <div class="col-sm-6">
                                         <p:selectOneMenu id="harvestingStyle"
-                                                         styleClass="form-control" value="#{harvestingClientsPage.newHarvestingStyle}"
-                                                         rendered="#{harvestingClientsPage.initialSettingsValidated}">
+                                                         binding="#{harvestingClientsPage.remoteArchiveStyleMenu}"
+                                                         styleClass="form-control" 
+                                                         value="#{harvestingClientsPage.newHarvestingStyle}"
+                                                         rendered="#{harvestingClientsPage.initialSettingsValidated}"
+                                                         validator="#{harvestingClientsPage.validateRemoteArchiveStyle}">
+                                            <f:selectItem itemLabel="#{bundle.select}" itemValue="" noSelectionOption="true" />
                                             <f:selectItems value="#{harvestingClientsPage.harvestingStylesSelectItems}" />
                                         </p:selectOneMenu>
                                         <p:message for="harvestingStyle"/>
@@ -547,7 +556,9 @@
                                                        value="#{bundle['harvestclients.viewEditDialog.btn.save']}"
                                                        update="newHarvestingClientDialogContent :messagePanel :harvestingClientsForm clientsTable emptyClientsTable"
                                                        actionListener="#{harvestingClientsPage.saveClient}"
-                                                       oncomplete="if (args &amp;&amp; !args.validationFailed) PF('newHarvestingClientForm').hide(); else PF('newHarvestingClientForm').show();bind_bsui_components();"/>
+                                                       oncomplete="if (args &amp;&amp; !args.validationFailed) PF('newHarvestingClientForm').hide(); else PF('newHarvestingClientForm').show();bind_bsui_components();">
+                                            <f:param name="DO_VALIDATION" value="true"/>
+                                        </p:commandLink>
                                         <button class="btn btn-link" jsf:rendered="#{harvestingClientsPage.initialSettingsValidated}" onclick="PF('newHarvestingClientForm').hide()" type="button">
                                             #{bundle.cancel}
                                         </button>