Import updateExisting parameter implemented

import-cmdtool/src/main/java/res/configuration.properties

@@ -124,6 +124,9 @@ ingest.startIndexer=true
 #when true, automatically sort relations in merged RELS-EXT datastreams (after incremental import of document parts)
 ingest.sortRelations=true
 
+#when true, existing foxml objects with the same PID are replaced with the newly imported; when false, existing objects are preserved - only RELS-EXT RDF relations from the newly imported objects are added to the existing ones
+ingest.updateExisting=false
+
 # connection to fedora repository (replication target)
 ingest.url=${fedoraHost}
 ingest.user=${fedoraUser}

import-cmdtool/src/main/resources/res/configuration.properties

@@ -124,6 +124,10 @@ ingest.startIndexer=true
 #when true, automatically sort relations in merged RELS-EXT datastreams (after incremental import of document parts)
 ingest.sortRelations=true
 
+#when true, existing foxml objects with the same PID are replaced with the newly imported; when false, existing objects are preserved - only RELS-EXT RDF relations from the newly imported objects are added to the existing ones
+ingest.updateExisting=false
+
+
 # connection to fedora repository (replication target)
 ingest.url=${fedoraHost}
 ingest.user=${fedoraUser}

import-mets/src/main/java/res/configuration.properties

@@ -119,6 +119,10 @@ ingest.startIndexer=true
 #when true, automatically sort relations in merged RELS-EXT datastreams (after incremental import of document parts)
 ingest.sortRelations=true
 
+#when true, existing foxml objects with the same PID are replaced with the newly imported; when false, existing objects are preserved - only RELS-EXT RDF relations from the newly imported objects are added to the existing ones
+ingest.updateExisting=false
+
+
 # connection to fedora repository (replication target)
 ingest.url=${fedoraHost}
 ingest.user=${fedoraUser}

import-mets/src/main/resources/res/configuration.properties

@@ -119,6 +119,10 @@ ingest.startIndexer=true
 #when true, automatically sort relations in merged RELS-EXT datastreams (after incremental import of document parts)
 ingest.sortRelations=true
 
+#when true, existing foxml objects with the same PID are replaced with the newly imported; when false, existing objects are preserved - only RELS-EXT RDF relations from the newly imported objects are added to the existing ones
+ingest.updateExisting=false
+
+
 # connection to fedora repository (replication target)
 ingest.url=${fedoraHost}
 ingest.user=${fedoraUser}

import/src/main/java/org/kramerius/Import.java

@@ -90,6 +90,11 @@ public static void ingest(final String url, final String user, final String pwd,
             log.info("INGEST CONFIGURED TO BE SKIPPED, RETURNING");
             return;
         }
+
+        boolean updateExisting = Boolean.valueOf (System.getProperties().containsKey("ingest.updateExisting") ? System.getProperty("ingest.updateExisting") : KConfiguration.getInstance().getConfiguration().getString("ingest.updateExisting", "false"));
+        log.info("INGEST updateExisting: "+updateExisting);
+
+
         long start = System.currentTimeMillis();
 
         File importFile = new File(importRoot);
@@ -104,7 +109,7 @@ public static void ingest(final String url, final String user, final String pwd,
         Set<TitlePidTuple> roots = new HashSet<TitlePidTuple>();
         Set<String> sortRelations = new HashSet<String>();
         if (importFile.isDirectory()) {
-            visitAllDirsAndFiles(importFile, roots, sortRelations);
+            visitAllDirsAndFiles(importFile, roots, sortRelations, updateExisting);
         } else {
             BufferedReader reader = null;
             try {
@@ -128,7 +133,7 @@ public static void ingest(final String url, final String user, final String pwd,
                         continue;
                     }
                     log.info("Importing " + importItem.getAbsolutePath());
-                    visitAllDirsAndFiles(importItem, roots, sortRelations);
+                    visitAllDirsAndFiles(importItem, roots, sortRelations, updateExisting);
                 }
                 reader.close();
             } catch (IOException e) {
@@ -196,7 +201,7 @@ protected PasswordAuthentication getPasswordAuthentication() {
         of = new ObjectFactory();
     }
 
-    private static void visitAllDirsAndFiles(File importFile, Set<TitlePidTuple> roots, Set<String> sortRelations) {
+    private static void visitAllDirsAndFiles(File importFile, Set<TitlePidTuple> roots, Set<String> sortRelations, boolean updateExisting) {
         if (importFile == null) {
             return;
         }
@@ -207,7 +212,7 @@ private static void visitAllDirsAndFiles(File importFile, Set<TitlePidTuple> roo
                 Arrays.sort(children);
             }
             for (int i = 0; i < children.length; i++) {
-                visitAllDirsAndFiles(children[i], roots, sortRelations);
+                visitAllDirsAndFiles(children[i], roots, sortRelations, updateExisting);
             }
         } else {
             DigitalObject dobj = null;
@@ -222,12 +227,12 @@ private static void visitAllDirsAndFiles(File importFile, Set<TitlePidTuple> roo
                 log.log(Level.FINE, "Underlying error was:", e);
                 return;
             }
-            ingest(importFile, dobj.getPID(), sortRelations, roots);
+            ingest(importFile, dobj.getPID(), sortRelations, roots, updateExisting);
             checkRoot(dobj, roots);
         }
     }
 
-    public static void ingest(InputStream is, String pid, Set<String> sortRelations, Set<TitlePidTuple> roots) throws IOException {
+    public static void ingest(InputStream is, String pid, Set<String> sortRelations, Set<TitlePidTuple> roots, boolean updateExisting) throws IOException {
 
         long start = System.currentTimeMillis();
         ByteArrayOutputStream bos = new ByteArrayOutputStream();
@@ -239,16 +244,39 @@ public static void ingest(InputStream is, String pid, Set<String> sortRelations,
 
             //if (sfex.getMessage().contains("ObjectExistsException")) {
             if (objectExists(pid)) {
-                log.info("Merging with existing object " + pid);
-                if (merge(bytes)){
-                    if (sortRelations != null) {
-                        sortRelations.add(pid);
-                        log.info("Added merged object for sorting relations:"+pid);
+                if (updateExisting){
+                    log.info("Replacing existing object " + pid);
+                    try{
+                        port.purgeObject(pid, "", false);
+                        log.info("purged old object "+pid);
+                    }catch(Exception ex){
+                        log.severe("Cannot purge object "+pid+", skipping: "+ex);
+                        throw new RuntimeException(ex);
+                    }
+                    try {
+                        port.ingest(bytes, "info:fedora/fedora-system:FOXML-1.1", "Initial ingest");
+                        log.info("Ingested new object "+pid);
+                    } catch (SOAPFaultException rsfex) {
+                        log.severe("Replace ingest SOAP fault:" + rsfex);
+                        throw new RuntimeException(rsfex);
                     }
-                    if(roots!= null ){
+                    if (roots != null) {
                         TitlePidTuple npt = new TitlePidTuple("", pid);
                         roots.add(npt);
-                        log.info("Added merged object for indexing:"+pid);
+                        log.info("Added replaced object for indexing:" + pid);
+                    }
+                }else {
+                    log.info("Merging with existing object " + pid);
+                    if (merge(bytes)) {
+                        if (sortRelations != null) {
+                            sortRelations.add(pid);
+                            log.info("Added merged object for sorting relations:" + pid);
+                        }
+                        if (roots != null) {
+                            TitlePidTuple npt = new TitlePidTuple("", pid);
+                            roots.add(npt);
+                            log.info("Added merged object for indexing:" + pid);
+                        }
                     }
                 }
             } else {
@@ -263,7 +291,7 @@ public static void ingest(InputStream is, String pid, Set<String> sortRelations,
         log.info("Ingested:" + pid + " in " + (System.currentTimeMillis() - start) + "ms, count:" + counter);
     }
 
-    public static void ingest(File file, String pid, Set<String> sortRelations, Set<TitlePidTuple> roots) {
+    public static void ingest(File file, String pid, Set<String> sortRelations, Set<TitlePidTuple> roots, boolean updateExisting) {
         if (pid == null) {
             try {
                 Object obj = unmarshaller.unmarshal(file);
@@ -277,7 +305,7 @@ public static void ingest(File file, String pid, Set<String> sortRelations, Set<
 
         try {
             FileInputStream is = new FileInputStream(file);
-            ingest(is, pid, sortRelations, roots);
+            ingest(is, pid, sortRelations, roots, updateExisting);
         } catch (Exception ex) {
             log.log(Level.SEVERE, "Ingestion error ", ex);
             throw new RuntimeException(ex);

import/src/main/java/org/kramerius/imports/ParametrizedImport.java

@@ -16,15 +16,14 @@
  */
 package org.kramerius.imports;
 
-import java.io.File;
-import java.io.IOException;
-import java.util.logging.Level;
-
-import org.kramerius.Import;
-
 import cz.incad.kramerius.processes.annotations.ParameterName;
 import cz.incad.kramerius.processes.annotations.Process;
 import cz.incad.kramerius.processes.impl.ProcessStarter;
+import org.kramerius.Import;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.logging.Level;
 
 /**
  * Parametrized import proces
@@ -39,10 +38,12 @@ public class ParametrizedImport {
 
     @Process
     public static void process( @ParameterName("importDirectory") File importDirectory, 
-            @ParameterName("startIndexer")Boolean startIndexer) {
+            @ParameterName("startIndexer")Boolean startIndexer,
+            @ParameterName("updateExisting")Boolean updateExisting) {
 
         System.setProperty("import.directory", importDirectory.getAbsolutePath());
         System.setProperty("ingest.startIndexer", startIndexer.toString());
+        System.setProperty("ingest.updateExisting", updateExisting.toString());
         System.setProperty("ingest.skip", "false");   //import se bude vždy spouštět
 
         try {

import/src/main/java/org/kramerius/imports/input/ParametrizedImportInputTemplate.java

@@ -16,36 +16,24 @@
  */
 package org.kramerius.imports.input;
 
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Writer;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Locale;
-import java.util.Map;
-import java.util.Properties;
-import java.util.ResourceBundle;
-import java.util.Set;
-
-import org.antlr.stringtemplate.StringTemplate;
-import org.antlr.stringtemplate.StringTemplateGroup;
-import org.antlr.stringtemplate.language.DefaultTemplateLexer;
-import org.kramerius.processes.filetree.TreeItem;
-import org.kramerius.processes.filetree.TreeModelFilter;
-import org.kramerius.processes.utils.BasicStringTemplateGroup;
-import org.kramerius.processes.utils.TreeModelUtils;
-
 import com.google.inject.Inject;
 import com.google.inject.Provider;
-
 import cz.incad.kramerius.processes.LRProcessDefinition;
 import cz.incad.kramerius.processes.template.ProcessInputTemplate;
 import cz.incad.kramerius.service.ResourceBundleService;
 import cz.incad.kramerius.utils.conf.KConfiguration;
 import cz.incad.kramerius.utils.stemplates.ResourceBundleUtils;
+import org.antlr.stringtemplate.StringTemplate;
+import org.antlr.stringtemplate.StringTemplateGroup;
+import org.antlr.stringtemplate.language.DefaultTemplateLexer;
+import org.kramerius.processes.filetree.TreeItem;
+import org.kramerius.processes.filetree.TreeModelFilter;
+import org.kramerius.processes.utils.TreeModelUtils;
+
+import java.io.*;
+import java.util.Locale;
+import java.util.Properties;
+import java.util.ResourceBundle;
 
 public class ParametrizedImportInputTemplate implements ProcessInputTemplate {
 
@@ -88,7 +76,11 @@ public boolean accept(File file) {
 
         Boolean startIndexer = configuration.getConfiguration().getBoolean("ingest.startIndexer");
         template.setAttribute("startIndexer",startIndexer);
-
+
+        Boolean updateExisting = configuration.getConfiguration().getBoolean("ingest.updateExisting");
+        template.setAttribute("updateExisting",updateExisting);
+
+
         writer.write(template.toString());
     }
 

import/src/main/java/org/kramerius/replications/SecondPhase.java

@@ -102,7 +102,7 @@ public void ingest(File foxmlfile) throws PhaseException{
         LOGGER.info("ingesting '"+foxmlfile.getAbsolutePath()+"'");
         Import.initialize(KConfiguration.getInstance().getProperty("ingest.user"), KConfiguration.getInstance().getProperty("ingest.password"));
         try {
-            Import.ingest(foxmlfile, null, null, null);  //TODO třetí parametr má být List<String>, inicializovaný na začátku této fáze a předaný třetí fázi, kde se budou třídit vazby
+            Import.ingest(foxmlfile, null, null, null, false);  //TODO třetí parametr má být List<String>, inicializovaný na začátku této fáze a předaný třetí fázi, kde se budou třídit vazby
         } catch (RuntimeException e) {
             if (e.getCause() != null) throw new PhaseException(this, e.getCause());
             else throw new PhaseException(this,e);

import/src/main/resources/org/kramerius/imports/input/parametrizedimport.stg

@@ -29,17 +29,18 @@ tree(root) ::=<<
 
 
 
-form(importDirectory,importRootDirectory,  startIndexer, bundle) ::=<<
+form(importDirectory,importRootDirectory,  startIndexer, updateExisting, bundle) ::=<<
 
 <div style="width:100%">
     <script language="JavaScript" type="text/javascript"> 
     <!--    
         // send parameters to server
         window.onProcessFormSend = function() {
             var startIndexer = \$('#startIndexer').attr('checked');
+            var updateExisting = \$('#updateExisting').attr('checked');
             var charsThatMustBeEscaped = [':',';','}','{','\\\\'];
             var vals = 'importDirectory='+\$('#importDirectory').val().escapeChars(charsThatMustBeEscaped)
-                +';startIndexer='+startIndexer
+                +';startIndexer='+startIndexer+';updateExisting='+updateExisting
                 ;
 
             var url = "lr?action=form_post&def=parametrizedimport&paramsMapping={"+vals+"}&out=text";
@@ -104,19 +105,23 @@ form(importDirectory,importRootDirectory,  startIndexer, bundle) ::=<<
 </fieldset>
 
 <!-- spustit indexer  -->
-$if(startIndexer)$
 <fieldset style="margin-top:10px;border: 1px solid gray;">
     <legend style="border:none">$bundle.("k3replication.othersettings.fields")$</legend>
-    <input type="checkbox" id="startIndexer" name="startIndexer" value="startIndexer" checked>$bundle.("import.indexerstart")$</input>
-</fieldset>
-$else$
-<fieldset style="margin-top:10px;border: 1px solid gray;">
-    <legend style="border:none">$bundle.("k3replication.othersettings.fields")$</legend>
-    <input type="checkbox" id="startIndexer" name="startIndexer" value="startIndexer">$bundle.("import.indexerstart")$</input>
+    $if(startIndexer)$
+        <input type="checkbox" id="startIndexer" name="startIndexer" value="startIndexer" checked>$bundle.("import.indexerstart")$</input>
+    $else$
+        <input type="checkbox" id="startIndexer" name="startIndexer" value="startIndexer">$bundle.("import.indexerstart")$</input>
+    $endif$
+    <br>
+    $if(updateExisting)$
+        <input type="checkbox" id="updateExisting" name="updateExisting" value="updateExisting" checked>$bundle.("import.updateExisting")$</input>
+    $else$
+        <input type="checkbox" id="updateExisting" name="updateExisting" value="updateExisting">$bundle.("import.updateExisting")$</input>
+    $endif$
 </fieldset>
-$endif$
 
-
+
+
     <div id="_selectImportFolderDialog" style="display:none">
         <h3>  $bundle.("import.selection.dialog")$</h3>
         <div id="_selectImportFolderTree">

search/src/java/labels.properties

@@ -1101,6 +1101,7 @@ import.user=User
 import.password=Password
 import.selection.dialog=Folder selection
 import.indexerstart=Start index process
+import.updateExisting=Replace existing objects
 
 administrator.menu.dialogs.parametrizedimport.title=Import FOXML...
 administrator.menu.dialogs.parametrizedconvert.title=Import from K3 format...

search/src/java/labels_cs.properties

@@ -1111,6 +1111,7 @@ import.user=U\u017Eivatel
 import.password=Heslo
 import.selection.dialog=V\u00FDb\u011Br adres\u00E1\u0159e
 import.indexerstart=Spustit indexa\u010Dn\u00ED proces
+import.updateExisting=Nahradit existující objekty
 
 administrator.menu.dialogs.parametrizedimport.title=Import FOXML...
 administrator.menu.dialogs.parametrizedconvert.title=Import z K3 form\u00E1tu...