From f716a766a26588eb68be147aaa34ec45c11b53e5 Mon Sep 17 00:00:00 2001 From: "pavel.stastny@gmail.com" Date: Tue, 10 Mar 2015 13:52:48 +0100 Subject: [PATCH 001/235] sources & cols --- .../cz/incad/kramerius/utils/DCUtils.java | 6 + .../CDKVirtualCollectionsGet.java | 14 ++ .../impl/CDKResourcesFilter.java | 46 ++++ .../impl/CDKVirtualCollectionsGetImpl.java | 223 ++++++++++++++++++ .../main/java/res/configuration.properties | 6 + .../impl/criteria/mw/DatesParserTest.java | 1 + .../impl/CDKVirtualCollectionsTest.java | 58 +++++ .../pdf/impl/drobnustky_pages_selection.xml | 2 +- .../pdf/impl/drobnustky_parent_first_page.xml | 2 +- ...rodni_listy_drobnustky_selection_pages.xml | 2 +- .../pdf/impl/narodni_listy_parent.xml | 2 +- .../impl/narodni_listy_selection_pages.xml | 2 +- .../kramerius/virtualcollections/impl/DC.xml | 8 + .../kramerius/virtualcollections/impl/vc.json | 9 + .../VirtualCollectionsResourceClient.java | 121 +++++----- .../rest/api/guice/ApiServletModule.java | 2 + .../admin/vc/VirtualCollectionsResource.java | 21 +- .../virtualcollection/ClientResources.java | 121 ++++++++++ .../ClientVirtualCollections.java | 32 ++- .../Kramerius/backend/guice/BaseModule.java | 5 + 20 files changed, 612 insertions(+), 71 deletions(-) create mode 100644 common/src/main/java/cz/incad/kramerius/virtualcollections/CDKVirtualCollectionsGet.java create mode 100644 common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKResourcesFilter.java create mode 100644 common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java create mode 100644 common/src/test/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsTest.java create mode 100644 common/src/test/resources/cz/incad/kramerius/virtualcollections/impl/DC.xml create mode 100644 common/src/test/resources/cz/incad/kramerius/virtualcollections/impl/vc.json create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientResources.java diff --git a/common/src/main/java/cz/incad/kramerius/utils/DCUtils.java b/common/src/main/java/cz/incad/kramerius/utils/DCUtils.java index adc2b2e20d..27ff543916 100644 --- a/common/src/main/java/cz/incad/kramerius/utils/DCUtils.java +++ b/common/src/main/java/cz/incad/kramerius/utils/DCUtils.java @@ -106,6 +106,12 @@ public static String dateFromDC(org.w3c.dom.Document dc) { else return null; } + public static String sourceFromDC(org.w3c.dom.Document dc) { + ArrayList dates = findElmTexts(dc, "source"); + if (!dates.isEmpty()) return dates.get(0); + else return null; + } + public static String languageFromDC(org.w3c.dom.Document dc) { ArrayList dates = findElmTexts(dc, "language"); if (!dates.isEmpty()) return dates.get(0); diff --git a/common/src/main/java/cz/incad/kramerius/virtualcollections/CDKVirtualCollectionsGet.java b/common/src/main/java/cz/incad/kramerius/virtualcollections/CDKVirtualCollectionsGet.java new file mode 100644 index 0000000000..a260c762ea --- /dev/null +++ b/common/src/main/java/cz/incad/kramerius/virtualcollections/CDKVirtualCollectionsGet.java @@ -0,0 +1,14 @@ +package cz.incad.kramerius.virtualcollections; + +import java.util.List; + +public interface CDKVirtualCollectionsGet { + + public List virtualCollectionsFromResource(String res); + + public VirtualCollection virtualCollectionsFromResource(String vc, String res); + + public List virtualCollections(); + + public String getResource(String vcId); +} diff --git a/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKResourcesFilter.java b/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKResourcesFilter.java new file mode 100644 index 0000000000..190d999749 --- /dev/null +++ b/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKResourcesFilter.java @@ -0,0 +1,46 @@ +package cz.incad.kramerius.virtualcollections.impl; + +import java.util.List; + +import cz.incad.kramerius.utils.conf.KConfiguration; +import cz.incad.kramerius.virtualcollections.VirtualCollection; + +public class CDKResourcesFilter { + + public List getResources() { + List al = KConfiguration.getInstance().getConfiguration().getList("cdk.collections.sources"); + return al; + } + + public List getHidden() { + List al = KConfiguration.getInstance().getConfiguration().getList("cdk.collections.hidden"); + return al; + } + + public boolean isResource(VirtualCollection vc) { + return isResource(vc.getPid()); + } + + public boolean isResource(String pid) { + List al = getResources(); + return al.contains(pid); + } + + public boolean isHidden(VirtualCollection vc) { + return isHidden(vc.getPid()); + } + + public boolean isHidden(String pid) { + List al = getHidden(); + return al.contains(pid); + } + + public boolean isFiltered(VirtualCollection vc) { + return isResource(vc.getPid()) || isHidden(vc.getPid()); + } + + public boolean isFiltered(String pid) { + return isResource(pid) || isHidden(pid); + } + +} diff --git a/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java b/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java new file mode 100644 index 0000000000..755bd1b5b4 --- /dev/null +++ b/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java @@ -0,0 +1,223 @@ +package cz.incad.kramerius.virtualcollections.impl; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.ws.rs.core.MediaType; +import javax.xml.xpath.XPathExpressionException; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.w3c.dom.Document; + +import com.google.inject.Inject; +import com.google.inject.name.Named; +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.WebResource; + +import cz.incad.kramerius.FedoraAccess; +import cz.incad.kramerius.utils.DCUtils; +import cz.incad.kramerius.virtualcollections.CDKVirtualCollectionsGet; +import cz.incad.kramerius.virtualcollections.VirtualCollection; +import cz.incad.kramerius.virtualcollections.VirtualCollection.CollectionDescription; + +public class CDKVirtualCollectionsGetImpl implements CDKVirtualCollectionsGet { + + private static Map> COLLECTIONS_MAPPING = new HashMap>(); + + private static final String URI_PREFIX = "URI"; + + public static final Logger LOGGER = Logger + .getLogger(CDKVirtualCollectionsGetImpl.class.getName()); + + @Inject + @Named("securedFedoraAccess") + FedoraAccess fedoraAccess; + + public CDKVirtualCollectionsGetImpl() { + } + + public List virtualCollectionsFromResource( + String resource) { + clearResource(resource); + List vcs = new ArrayList(); + CDKResourcesFilter filter = new CDKResourcesFilter(); + if (!filter.isHidden(resource)) { + try { + Document dc = this.fedoraAccess.getDC(resource); + String vcpoint = appendVCPoint(disectURL(dc)); + JSONArray jsonArray = virtualCollectionsFromPoint(vcpoint); + for (int i = 0, ll = jsonArray.length(); i < ll; i++) { + VirtualCollection vc = collectionFromJSON(jsonArray + .getJSONObject(i)); + addResourceVC(resource, vc.getPid()); + vcs.add(vc); + } + } catch (IOException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } catch (XPathExpressionException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } catch (JSONException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } + } + return vcs; + } + + public List virtualCollections() { + clearAllResources(); + List vcs = new ArrayList(); + CDKResourcesFilter filter = new CDKResourcesFilter(); + List resources = filter.getResources(); + for (String res : resources) { + List subVCS = virtualCollectionsFromResource(res); + vcs.addAll(subVCS); + } + return vcs; + } + + @Override + public VirtualCollection virtualCollectionsFromResource(String vc, + String res) { + CDKResourcesFilter filter = new CDKResourcesFilter(); + if (filter.isResource(res)) { + if (!filter.isHidden(res)) { + // http://krameriusdemo.mzk.cz/search/api/v5.0/vc/vc:758d7168-d625-4648-911a-9a80473a1717 + try { + Document dc = this.fedoraAccess.getDC(res); + String vcpoint = appendVCPoint(disectURL(dc)) + "/" + vc; + JSONObject jsonObject = virtualCollectionFromPoint(vcpoint, + vc); + VirtualCollection vcObject = collectionFromJSON(jsonObject); + addResourceVC(res, vcObject.getPid()); + return vcObject; + } catch (IOException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } catch (XPathExpressionException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } catch (JSONException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } + } + } + return null; + } + + @Override + public String getResource(String vcId) { + Set keys = COLLECTIONS_MAPPING.keySet(); + for (String resource : keys) { + List list = COLLECTIONS_MAPPING.get(resource); + if (list.contains(vcId)) + return resource; + } + return null; + } + + + public FedoraAccess getFedoraAccess() { + return fedoraAccess; + } + + public void setFedoraAccess(FedoraAccess fedoraAccess) { + this.fedoraAccess = fedoraAccess; + } + + static VirtualCollection collectionFromJSON(JSONObject jsonObject) + throws JSONException { + String pid = jsonObject.getString("pid"); + String label = jsonObject.getString("label"); + boolean cleave = jsonObject.getBoolean("canLeave"); + VirtualCollection col = new VirtualCollection(label, pid, cleave); + JSONObject descs = jsonObject.getJSONObject("descs"); + Iterator keys = descs.keys(); + while (keys.hasNext()) { + String k = (String) keys.next(); + col.addDescription(k, descs.getString(k)); + } + return col; + } + + static JSONArray virtualCollectionsFromPoint(String point) + throws JSONException { + Client c = Client.create(); + WebResource r = c.resource(point); + String t = r.accept(MediaType.APPLICATION_JSON).get(String.class); + JSONArray jsonArr = new JSONArray(t); + return jsonArr; + } + + static JSONObject virtualCollectionFromPoint(String point, String pid) + throws JSONException { + Client c = Client.create(); + WebResource r = c.resource(point + "/" + pid); + String t = r.accept(MediaType.APPLICATION_JSON).get(String.class); + JSONObject json = new JSONObject(t); + return json; + } + + static String appendVCPoint(String baseUrl) { + if (!baseUrl.endsWith("/")) { + baseUrl += "/"; + } + return baseUrl + "api/v5.0/vc"; + } + + public static String disectURL(Document dc) throws XPathExpressionException { + String src = DCUtils.sourceFromDC(dc); + return src; + } + + static synchronized void clearAllResources() { + COLLECTIONS_MAPPING.clear(); + } + + static synchronized void clearResource(String resource) { + COLLECTIONS_MAPPING.remove(resource); + } + + static synchronized void addResourceVC(String resource, String vcid) { + if (!COLLECTIONS_MAPPING.containsKey(resource)) { + COLLECTIONS_MAPPING.put(resource, new ArrayList()); + } + List alist = COLLECTIONS_MAPPING.get(resource); + if (!alist.contains(vcid)) { + alist.add(vcid); + } + } + + static synchronized void removeResourceVC(String resource, String vcid) { + if (!COLLECTIONS_MAPPING.containsKey(resource)) { + COLLECTIONS_MAPPING.put(resource, new ArrayList()); + } + List alist = COLLECTIONS_MAPPING.get(resource); + if (alist.contains(vcid)) { + alist.remove(vcid); + } + } + + public static void main(String[] args) throws JSONException { + List cols = new ArrayList(); + String str = "http://kramerius4.nkp.cz/search/"; + JSONArray jsonArray = virtualCollectionsFromPoint(appendVCPoint(str)); + for (int i = 0, ll = jsonArray.length(); i < ll; i++) { + cols.add(collectionFromJSON(jsonArray.getJSONObject(i))); + } + + for (VirtualCollection vc : cols) { + System.out.println(vc.getLabel()); + System.out.println(vc.getPid()); + System.out.println(vc.getDescriptionLocale("en")); + System.out.println(vc.getDescriptionLocale("cs")); + } + } + +} diff --git a/common/src/main/java/res/configuration.properties b/common/src/main/java/res/configuration.properties index 51bfb4f101..0f495fc1a5 100644 --- a/common/src/main/java/res/configuration.properties +++ b/common/src/main/java/res/configuration.properties @@ -397,3 +397,9 @@ securedstreams=TEXT_OCR #určuje zda proces mazání foxml objekty zcela smaže (true) nebo jen nastaví jejich stav jako Deleted delete.purgeObjects=true + + +## CDK resources +cdk.collections.sources=vc:44679769-b5bb-4ac7-ad27-a0c44698c2ea,vc:f750b424-bda4-4113-849a-5e9dbbfb5846,vc:700a6cc7-1e5c-4487-b111-c425cbc51091,vc:c4bb27af-3a51-4ac2-95c7-fd393b489e26,vc:9d2195b1-5403-4b3d-ab7c-680ac442bb73 +cdk.collections.hidden=vc:9d2195b1-5403-4b3d-ab7c-680ac442bb73 + diff --git a/common/src/test/java/cz/incad/kramerius/security/impl/criteria/mw/DatesParserTest.java b/common/src/test/java/cz/incad/kramerius/security/impl/criteria/mw/DatesParserTest.java index 9918bb9be7..a6a3d505e4 100644 --- a/common/src/test/java/cz/incad/kramerius/security/impl/criteria/mw/DatesParserTest.java +++ b/common/src/test/java/cz/incad/kramerius/security/impl/criteria/mw/DatesParserTest.java @@ -29,6 +29,7 @@ import antlr.TokenStreamException; public class DatesParserTest { + @Test public void shouldPass() throws RecognitionException, TokenStreamException { diff --git a/common/src/test/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsTest.java b/common/src/test/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsTest.java new file mode 100644 index 0000000000..1523748a45 --- /dev/null +++ b/common/src/test/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsTest.java @@ -0,0 +1,58 @@ +package cz.incad.kramerius.virtualcollections.impl; + +import java.io.IOException; +import java.net.URL; +import java.nio.charset.Charset; +import java.util.Map; +import java.util.Set; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.xpath.XPathExpressionException; + +import junit.framework.Assert; + +import org.json.JSONException; +import org.json.JSONObject; +import org.junit.Test; +import org.w3c.dom.Document; +import org.xml.sax.SAXException; + +import cz.incad.kramerius.utils.IOUtils; +import cz.incad.kramerius.utils.XMLUtils; +import cz.incad.kramerius.virtualcollections.VirtualCollection; +import cz.incad.kramerius.virtualcollections.impl.CDKVirtualCollectionsGetImpl; + +public class CDKVirtualCollectionsTest { + + @Test + public void testDisect() throws XPathExpressionException, ParserConfigurationException, SAXException, IOException { + URL urlRes = CDKVirtualCollectionsTest.class.getResource("DC.xml"); + Document document = XMLUtils.parseDocument(urlRes.openStream(),true); + String url = CDKVirtualCollectionsGetImpl.disectURL(document); + Assert.assertEquals("http://localhost:8080/search", url); + } + + @Test + public void testAppendVCPoint() throws ParserConfigurationException, SAXException, IOException, XPathExpressionException { + URL urlRes = CDKVirtualCollectionsTest.class.getResource("DC.xml"); + Document document = XMLUtils.parseDocument(urlRes.openStream(),true); + String vcpoint = CDKVirtualCollectionsGetImpl.appendVCPoint(CDKVirtualCollectionsGetImpl.disectURL(document)); + Assert.assertEquals("http://localhost:8080/search/api/v5.0/vc", vcpoint); + } + + @Test + public void testVCFromJSON() throws IOException, JSONException { + URL urlRes = CDKVirtualCollectionsTest.class.getResource("vc.json"); + String str = IOUtils.readAsString(urlRes.openStream(), Charset.forName("UTF-8"), true); + JSONObject jobj = new JSONObject(str); + VirtualCollection vc = CDKVirtualCollectionsGetImpl.collectionFromJSON(jobj); + Assert.assertEquals(vc.getPid(), "vc:f73dee31-ae76-4dbc-b7b9-d986df497596"); + Assert.assertEquals(vc.getLabel(), "vc:f73dee31-ae76-4dbc-b7b9-d986df497596"); + Assert.assertEquals(vc.isCanLeave(), true); + Map map = vc.getDescriptionsMap(); + Assert.assertTrue(map.size() == 2); + Assert.assertTrue(map.containsKey("en")); + Assert.assertTrue(map.containsKey("cs")); + + } +} diff --git a/common/src/test/resources/cz/incad/kramerius/pdf/impl/drobnustky_pages_selection.xml b/common/src/test/resources/cz/incad/kramerius/pdf/impl/drobnustky_pages_selection.xml index ddeb833c3f..71eb8ea637 100644 --- a/common/src/test/resources/cz/incad/kramerius/pdf/impl/drobnustky_pages_selection.xml +++ b/common/src/test/resources/cz/incad/kramerius/pdf/impl/drobnustky_pages_selection.xml @@ -1,7 +1,7 @@ - Kramerius 4 + Kramerius 5 diff --git a/common/src/test/resources/cz/incad/kramerius/pdf/impl/drobnustky_parent_first_page.xml b/common/src/test/resources/cz/incad/kramerius/pdf/impl/drobnustky_parent_first_page.xml index 9384442a29..8a330d7523 100644 --- a/common/src/test/resources/cz/incad/kramerius/pdf/impl/drobnustky_parent_first_page.xml +++ b/common/src/test/resources/cz/incad/kramerius/pdf/impl/drobnustky_parent_first_page.xml @@ -1,7 +1,7 @@ - Kramerius 4 + Kramerius 5 diff --git a/common/src/test/resources/cz/incad/kramerius/pdf/impl/narodni_listy_drobnustky_selection_pages.xml b/common/src/test/resources/cz/incad/kramerius/pdf/impl/narodni_listy_drobnustky_selection_pages.xml index 6477be4af1..6f43d79e11 100644 --- a/common/src/test/resources/cz/incad/kramerius/pdf/impl/narodni_listy_drobnustky_selection_pages.xml +++ b/common/src/test/resources/cz/incad/kramerius/pdf/impl/narodni_listy_drobnustky_selection_pages.xml @@ -1,7 +1,7 @@ - Kramerius 4 + Kramerius 5 diff --git a/common/src/test/resources/cz/incad/kramerius/pdf/impl/narodni_listy_parent.xml b/common/src/test/resources/cz/incad/kramerius/pdf/impl/narodni_listy_parent.xml index 4fff43bbb5..2eadc7acd4 100644 --- a/common/src/test/resources/cz/incad/kramerius/pdf/impl/narodni_listy_parent.xml +++ b/common/src/test/resources/cz/incad/kramerius/pdf/impl/narodni_listy_parent.xml @@ -1,7 +1,7 @@ - Kramerius 4 + Kramerius 5 diff --git a/common/src/test/resources/cz/incad/kramerius/pdf/impl/narodni_listy_selection_pages.xml b/common/src/test/resources/cz/incad/kramerius/pdf/impl/narodni_listy_selection_pages.xml index 37a06af10f..f40297528d 100644 --- a/common/src/test/resources/cz/incad/kramerius/pdf/impl/narodni_listy_selection_pages.xml +++ b/common/src/test/resources/cz/incad/kramerius/pdf/impl/narodni_listy_selection_pages.xml @@ -1,7 +1,7 @@ - Kramerius 4 + Kramerius 5 diff --git a/common/src/test/resources/cz/incad/kramerius/virtualcollections/impl/DC.xml b/common/src/test/resources/cz/incad/kramerius/virtualcollections/impl/DC.xml new file mode 100644 index 0000000000..192f0aa88a --- /dev/null +++ b/common/src/test/resources/cz/incad/kramerius/virtualcollections/impl/DC.xml @@ -0,0 +1,8 @@ + + + nejnovejsi + vc:6953c861-f2e3-4654-bb63-69b80ed98cec + URI:http://localhost:8080/search + canLeave:true + http://localhost:8080/search + diff --git a/common/src/test/resources/cz/incad/kramerius/virtualcollections/impl/vc.json b/common/src/test/resources/cz/incad/kramerius/virtualcollections/impl/vc.json new file mode 100644 index 0000000000..1b9b3289c0 --- /dev/null +++ b/common/src/test/resources/cz/incad/kramerius/virtualcollections/impl/vc.json @@ -0,0 +1,9 @@ + { + "pid": "vc:f73dee31-ae76-4dbc-b7b9-d986df497596", + "label": "vc:f73dee31-ae76-4dbc-b7b9-d986df497596", + "canLeave": true, + "descs": { + "en": "ACADEMIA", + "cs": "AKADEMIA" + } +} \ No newline at end of file diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/client/v50/client/VirtualCollectionsResourceClient.java b/rest/src/main/java/cz/incad/kramerius/rest/api/client/v50/client/VirtualCollectionsResourceClient.java index d05edd5ed7..1bb50a7773 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/client/v50/client/VirtualCollectionsResourceClient.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/client/v50/client/VirtualCollectionsResourceClient.java @@ -23,87 +23,100 @@ /** * Inforace o virtualni sbirce + * * @author pavels * */ public class VirtualCollectionsResourceClient { - - /** - * Seznam vsech virtualnich sbirek - * @return - */ - public static String vcs() { + + /** + * Seznam vsech virtualnich sbirek + * + * @return + */ + public static String vcs() { Client c = Client.create(); WebResource r = c.resource("http://localhost:8080/search/api/v5.0/vc"); String t = r.accept(MediaType.APPLICATION_JSON).get(String.class); return t; - } + } - /** - * Konkretni virtualni sbirka - * @param vcpid - * @return - */ - public static String vc(String vcpid) { + /** + * Konkretni virtualni sbirka + * + * @param vcpid + * @return + */ + public static String vc(String vcpid) { Client c = Client.create(); - WebResource r = c.resource("http://localhost:8080/search/api/v5.0/vc/"+vcpid); + WebResource r = c.resource("http://localhost:8080/search/api/v5.0/vc/" + + vcpid); String t = r.accept(MediaType.APPLICATION_JSON).get(String.class); return t; - } + } - /** - * Info o objektu - * @param vcpid - * @return - */ - public static String vcAsFedoraObject(String vcpid) { + /** + * Info o objektu + * + * @param vcpid + * @return + */ + public static String vcAsFedoraObject(String vcpid) { Client c = Client.create(); - WebResource r = c.resource("http://localhost:8080/search/api/v5.0/item/"+vcpid); + WebResource r = c + .resource("http://localhost:8080/search/api/v5.0/item/" + vcpid); String t = r.accept(MediaType.APPLICATION_JSON).get(String.class); return t; - } + } - /** - * Seznam vsech streamu objektu reprezentujici virtualni sbirku - * @param vcpid - * @return - */ - public static String vcAsStreamsObject(String vcpid) { + /** + * Seznam vsech streamu objektu reprezentujici virtualni sbirku + * + * @param vcpid + * @return + */ + public static String vcAsStreamsObject(String vcpid) { Client c = Client.create(); - WebResource r = c.resource("http://localhost:8080/search/api/v5.0/item/"+vcpid+"/streams"); + WebResource r = c + .resource("http://localhost:8080/search/api/v5.0/item/" + vcpid + + "/streams"); String t = r.accept(MediaType.APPLICATION_JSON).get(String.class); return t; - } + } - /** - * Ziskani streamu z objektu virtualni sbirky - * @param vcpid - * @param str - * @return - */ - public static String vcAsStreamObject(String vcpid, String str) { + /** + * Ziskani streamu z objektu virtualni sbirky + * + * @param vcpid + * @param str + * @return + */ + public static String vcAsStreamObject(String vcpid, String str) { Client c = Client.create(); - WebResource r = c.resource("http://localhost:8080/search/api/v5.0/item/"+vcpid+"/streams/"+str); + WebResource r = c + .resource("http://localhost:8080/search/api/v5.0/item/" + vcpid + + "/streams/" + str); String t = r.accept(MediaType.APPLICATION_JSON).get(String.class); return t; - } + } + + public static void main(String[] args) { + String vcs = vcs(); + System.out.println(vcs); + + String vc = vc("vc:f73dee31-ae76-4dbc-b7b9-d986df497596"); + System.out.println(vc); - public static void main(String[] args) { - String vcs = vcs(); - System.out.println(vcs); - - String vc = vc("vc:f73dee31-ae76-4dbc-b7b9-d986df497596"); - System.out.println(vc); + String vcaf = vcAsFedoraObject("vc:f73dee31-ae76-4dbc-b7b9-d986df497596"); + System.out.println(vcaf); - String vcaf = vcAsFedoraObject("vc:f73dee31-ae76-4dbc-b7b9-d986df497596"); - System.out.println(vcaf); + String vcStreams = vcAsStreamsObject("vc:f73dee31-ae76-4dbc-b7b9-d986df497596"); + System.out.println(vcStreams); - String vcStreams = vcAsStreamsObject("vc:f73dee31-ae76-4dbc-b7b9-d986df497596"); - System.out.println(vcStreams); + String vcDCSteram = vcAsStreamObject( + "vc:f73dee31-ae76-4dbc-b7b9-d986df497596", "DC"); + System.out.println(vcDCSteram); - String vcDCSteram = vcAsStreamObject("vc:f73dee31-ae76-4dbc-b7b9-d986df497596","DC"); - System.out.println(vcDCSteram); + } - } - } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java index 0940c50bf7..5ed22e1ec2 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java @@ -62,6 +62,7 @@ import cz.incad.kramerius.rest.api.k5.client.rights.ClientRightsResource; import cz.incad.kramerius.rest.api.k5.client.search.SearchResource; import cz.incad.kramerius.rest.api.k5.client.user.ClientUserResource; +import cz.incad.kramerius.rest.api.k5.client.virtualcollection.ClientResources; import cz.incad.kramerius.rest.api.k5.client.virtualcollection.ClientVirtualCollections; import cz.incad.kramerius.rest.api.processes.LRResource; import cz.incad.kramerius.rest.api.replication.CDKReplicationsResource; @@ -89,6 +90,7 @@ protected void configureServlets() { bind(ItemResource.class); bind(FeederResource.class); bind(ClientVirtualCollections.class); + bind(ClientResources.class); bind(SearchResource.class); bind(ClientRightsResource.class); bind(PDFResource.class); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/admin/vc/VirtualCollectionsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/admin/vc/VirtualCollectionsResource.java index f89dd701ab..d93d2b94fb 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/admin/vc/VirtualCollectionsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/admin/vc/VirtualCollectionsResource.java @@ -54,6 +54,10 @@ import cz.incad.kramerius.security.User; import cz.incad.kramerius.virtualcollections.VirtualCollection; import cz.incad.kramerius.virtualcollections.VirtualCollectionsManager; +import cz.incad.kramerius.virtualcollections.impl.CDKResourcesFilter; + + +// TODO: Change it @Path("/v5.0/admin/vc") public class VirtualCollectionsResource { @@ -71,6 +75,8 @@ public class VirtualCollectionsResource { @Inject Provider userProvider; + private CDKResourcesFilter cdkResFilter = new CDKResourcesFilter(); + @POST @Consumes(MediaType.APPLICATION_JSON + ";charset=utf-8") @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") @@ -131,6 +137,7 @@ public Response put(JSONObject jsonObj) { VirtualCollection vc = findVirtualCollection(this.fedoraAccess, pid); if (vc != null) { + String label = jsonObj.getString("label"); boolean canLeaveFlag = jsonObj.getBoolean("canLeave"); VirtualCollectionsManager.modify(pid, label, canLeaveFlag, @@ -167,7 +174,7 @@ public Response put(JSONObject jsonObj) { public Response delete(@PathParam("pid") String pid) { if (permit(this.userProvider.get())) { VirtualCollection vc = findVirtualCollection(this.fedoraAccess, pid); - if (vc != null) { + if (vc != null && (!this.cdkResFilter.isFiltered(vc.getPid()))) { try { VirtualCollectionsManager .deleteWOIndexer(pid, fedoraAccess); @@ -194,8 +201,12 @@ public Response oneVirtualCollection(@PathParam("pid") String pid) { VirtualCollection vc = findVirtualCollection(this.fedoraAccess, pid); if (vc != null) { - return Response.ok().entity(virtualCollectionTOJSON(vc)) - .build(); + if (!this.cdkResFilter.isFiltered(vc.getPid())) { + return Response.ok().entity(virtualCollectionTOJSON(vc)) + .build(); + } else { + throw new ObjectNotFound("cannot find vc '" + pid + "'"); + } } else { throw new ObjectNotFound("cannot find vc '" + pid + "'"); } @@ -217,7 +228,9 @@ public Response get() { new ArrayList()); JSONArray jsonArr = new JSONArray(); for (VirtualCollection vc : vcs) { - jsonArr.add(virtualCollectionTOJSON(vc)); + if (!this.cdkResFilter.isFiltered(vc.getPid())) { + jsonArr.add(virtualCollectionTOJSON(vc)); + } } return Response.ok().entity(jsonArr.toString()).build(); } catch (Exception e) { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientResources.java b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientResources.java new file mode 100644 index 0000000000..47ba042b43 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientResources.java @@ -0,0 +1,121 @@ +package cz.incad.kramerius.rest.api.k5.client.virtualcollection; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.ws.rs.Consumes; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.xml.xpath.XPathExpressionException; + +import org.w3c.dom.Document; + +import net.sf.json.JSONArray; +import net.sf.json.JSONObject; + +import com.google.inject.Inject; +import com.google.inject.name.Named; + +import cz.incad.kramerius.FedoraAccess; +import cz.incad.kramerius.rest.api.exceptions.GenericApplicationException; +import cz.incad.kramerius.rest.api.k5.admin.vc.VirtualCollectionsResource; +import cz.incad.kramerius.rest.api.replication.exceptions.ObjectNotFound; +import cz.incad.kramerius.virtualcollections.VirtualCollection; +import cz.incad.kramerius.virtualcollections.VirtualCollectionsManager; +import cz.incad.kramerius.virtualcollections.impl.CDKResourcesFilter; +import cz.incad.kramerius.virtualcollections.impl.CDKVirtualCollectionsGetImpl; + +@Path("/v5.0/sources") +public class ClientResources { + + public static Logger LOGGER = Logger.getLogger(ClientResources.class.getName()); + + @Inject + VirtualCollectionsManager manager; + + @Inject + @Named("securedFedoraAccess") + FedoraAccess fedoraAccess; + + private CDKResourcesFilter cdkResFilter = new CDKResourcesFilter(); + + + @GET + @Path("{pid}") + @Consumes + @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") + public Response oneVirtualCollection(@PathParam("pid") String pid) { + try { + VirtualCollection vc = VirtualCollectionsResource + .findVirtualCollection(this.fedoraAccess, pid); + if (vc != null && this.cdkResFilter.isResource(vc.getPid())) { + if (!this.cdkResFilter.isHidden(vc.getPid())) { + return Response + .ok() + .entity(resourceTOJSON(this.fedoraAccess, vc)).build(); + } else { + throw new ObjectNotFound("cannot find vc '" + pid + "'"); + } + } else { + throw new ObjectNotFound("cannot find vc '" + pid + "'"); + } + } catch (ObjectNotFound e) { + throw e; + } catch (Exception e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new GenericApplicationException(e.getMessage()); + } + } + + @GET + @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") + public Response get() { + try { + JSONArray jsonArr = new JSONArray(); + List resources = this.cdkResFilter.getResources(); + for (String pid : resources) { + if (!this.cdkResFilter.isHidden(pid)) { + VirtualCollection vc = VirtualCollectionsResource + .findVirtualCollection(this.fedoraAccess, pid); + if (vc != null) { + jsonArr.add(resourceTOJSON(this.fedoraAccess, vc)); + } + } + } + return Response.ok().entity(jsonArr.toString()).build(); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new GenericApplicationException(e.getMessage()); + } + } + + public static JSONObject resourceTOJSON(FedoraAccess fa, VirtualCollection vc) throws XPathExpressionException, IOException { + JSONObject jsonObj = new JSONObject(); + jsonObj.put("pid", vc.getPid()); + jsonObj.put("label", vc.getLabel()); + + Document dc = fa.getDC(vc.getPid()); + String url = CDKVirtualCollectionsGetImpl.disectURL(dc); + if (url != null) { + jsonObj.put("url", url); + } + + JSONObject jsonMap = new JSONObject(); + Map descMAp = vc.getDescriptionsMap(); + for (String k : descMAp.keySet()) { + jsonMap.put(k, descMAp.get(k)); + } + jsonObj.put("descs", jsonMap); + + return jsonObj; + } + +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientVirtualCollections.java b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientVirtualCollections.java index af053d6220..27552074b3 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientVirtualCollections.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientVirtualCollections.java @@ -40,8 +40,10 @@ import cz.incad.kramerius.rest.api.k5.admin.vc.VirtualCollectionsResource; import cz.incad.kramerius.rest.api.replication.exceptions.ObjectNotFound; import cz.incad.kramerius.security.utils.PasswordDigest; +import cz.incad.kramerius.virtualcollections.CDKVirtualCollectionsGet; import cz.incad.kramerius.virtualcollections.VirtualCollection; import cz.incad.kramerius.virtualcollections.VirtualCollection.CollectionDescription; +import cz.incad.kramerius.virtualcollections.impl.CDKResourcesFilter; import cz.incad.kramerius.virtualcollections.VirtualCollectionsManager; import javax.ws.rs.PathParam; @@ -59,21 +61,37 @@ public class ClientVirtualCollections { @Named("securedFedoraAccess") FedoraAccess fedoraAccess; + @Inject + CDKVirtualCollectionsGet cdkVirtGet; + + + @GET @Path("{pid}") @Consumes @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") public Response oneVirtualCollection(@PathParam("pid") String pid) { try { - VirtualCollection vc = VirtualCollectionsResource - .findVirtualCollection(this.fedoraAccess, pid); - if (vc != null) { + String res = this.cdkVirtGet.getResource(pid); + if (res == null) { + List vcs = this.cdkVirtGet.virtualCollections(); + for (VirtualCollection vc : vcs) { + if (vc.getPid().equals(pid)) { + return Response + .ok() + .entity(VirtualCollectionsResource + .virtualCollectionTOJSON(vc)).build(); + + } + } + throw new ObjectNotFound("cannot find vc '" + pid + "'"); + } else { + VirtualCollection vc = this.cdkVirtGet.virtualCollectionsFromResource(pid, res); return Response .ok() .entity(VirtualCollectionsResource .virtualCollectionTOJSON(vc)).build(); - } else { - throw new ObjectNotFound("cannot find vc '" + pid + "'"); + } } catch (ObjectNotFound e) { throw e; @@ -87,9 +105,7 @@ public Response oneVirtualCollection(@PathParam("pid") String pid) { @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") public Response get() { try { - List vcs = VirtualCollectionsManager - .getVirtualCollections(fedoraAccess, - new ArrayList()); + List vcs = this.cdkVirtGet.virtualCollections(); JSONArray jsonArr = new JSONArray(); for (VirtualCollection vc : vcs) { jsonArr.add(VirtualCollectionsResource diff --git a/search/src/java/cz/incad/Kramerius/backend/guice/BaseModule.java b/search/src/java/cz/incad/Kramerius/backend/guice/BaseModule.java index 7bf66d1412..bae88aec36 100644 --- a/search/src/java/cz/incad/Kramerius/backend/guice/BaseModule.java +++ b/search/src/java/cz/incad/Kramerius/backend/guice/BaseModule.java @@ -34,7 +34,9 @@ import cz.incad.kramerius.statistics.StatisticsAccessLog; import cz.incad.kramerius.statistics.impl.*; import cz.incad.kramerius.utils.conf.KConfiguration; +import cz.incad.kramerius.virtualcollections.CDKVirtualCollectionsGet; import cz.incad.kramerius.virtualcollections.VirtualCollection; +import cz.incad.kramerius.virtualcollections.impl.CDKVirtualCollectionsGetImpl; import javax.servlet.jsp.jstl.fmt.LocalizationContext; @@ -86,6 +88,9 @@ protected void configure() { Multibinder lfhooks = Multibinder.newSetBinder(binder(), LifeCycleHook.class); lfhooks.addBinding().to(AudioLifeCycleHook.class); + + // only CDK + bind(CDKVirtualCollectionsGet.class).to(CDKVirtualCollectionsGetImpl.class); } @Provides From 31b3be0bb9952ede003db512416d731873027747 Mon Sep 17 00:00:00 2001 From: "pavel.stastny@gmail.com" Date: Thu, 12 Mar 2015 14:44:34 +0100 Subject: [PATCH 002/235] error catch --- .../impl/CDKVirtualCollectionsGetImpl.java | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java b/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java index 755bd1b5b4..614a0f9073 100644 --- a/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java +++ b/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java @@ -21,6 +21,8 @@ import com.google.inject.Inject; import com.google.inject.name.Named; import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientHandlerException; +import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.FedoraAccess; @@ -148,11 +150,19 @@ static VirtualCollection collectionFromJSON(JSONObject jsonObject) static JSONArray virtualCollectionsFromPoint(String point) throws JSONException { - Client c = Client.create(); - WebResource r = c.resource(point); - String t = r.accept(MediaType.APPLICATION_JSON).get(String.class); - JSONArray jsonArr = new JSONArray(t); - return jsonArr; + try { + Client c = Client.create(); + WebResource r = c.resource(point); + String t = r.accept(MediaType.APPLICATION_JSON).get(String.class); + JSONArray jsonArr = new JSONArray(t); + return jsonArr; + } catch (UniformInterfaceException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + + } catch (ClientHandlerException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } + return new JSONArray(); } static JSONObject virtualCollectionFromPoint(String point, String pid) From dd32bc34d6cea2ebe16383cac85707144db64ead Mon Sep 17 00:00:00 2001 From: "pavel.stastny@gmail.com" Date: Thu, 12 Mar 2015 14:55:32 +0100 Subject: [PATCH 003/235] VC try, catch block --- .../virtualcollections/impl/CDKVirtualCollectionsGetImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java b/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java index 614a0f9073..91e620acd8 100644 --- a/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java +++ b/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java @@ -216,7 +216,8 @@ static synchronized void removeResourceVC(String resource, String vcid) { public static void main(String[] args) throws JSONException { List cols = new ArrayList(); - String str = "http://kramerius4.nkp.cz/search/"; + String str = "http://kramerius.lib.cas.cz/search/"; + //http://kramerius.lib.cas.cz/search/api/v5.0/vc JSONArray jsonArray = virtualCollectionsFromPoint(appendVCPoint(str)); for (int i = 0, ll = jsonArray.length(); i < ll; i++) { cols.add(collectionFromJSON(jsonArray.getJSONObject(i))); From 7da0ae941729e3a1c33c76b01ca6f1b438349666 Mon Sep 17 00:00:00 2001 From: "pavel.stastny@gmail.com" Date: Thu, 12 Mar 2015 15:42:04 +0100 Subject: [PATCH 004/235] Follow redirect --- .../impl/CDKVirtualCollectionsGetImpl.java | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java b/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java index 91e620acd8..39f6cfba5b 100644 --- a/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java +++ b/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java @@ -24,6 +24,7 @@ import com.sun.jersey.api.client.ClientHandlerException; import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.ClientConfig; import cz.incad.kramerius.FedoraAccess; import cz.incad.kramerius.utils.DCUtils; @@ -153,6 +154,7 @@ static JSONArray virtualCollectionsFromPoint(String point) try { Client c = Client.create(); WebResource r = c.resource(point); + r.setProperty(ClientConfig.PROPERTY_FOLLOW_REDIRECTS, new Boolean(true)); String t = r.accept(MediaType.APPLICATION_JSON).get(String.class); JSONArray jsonArr = new JSONArray(t); return jsonArr; @@ -214,21 +216,21 @@ static synchronized void removeResourceVC(String resource, String vcid) { } } - public static void main(String[] args) throws JSONException { - List cols = new ArrayList(); - String str = "http://kramerius.lib.cas.cz/search/"; - //http://kramerius.lib.cas.cz/search/api/v5.0/vc - JSONArray jsonArray = virtualCollectionsFromPoint(appendVCPoint(str)); - for (int i = 0, ll = jsonArray.length(); i < ll; i++) { - cols.add(collectionFromJSON(jsonArray.getJSONObject(i))); - } - - for (VirtualCollection vc : cols) { - System.out.println(vc.getLabel()); - System.out.println(vc.getPid()); - System.out.println(vc.getDescriptionLocale("en")); - System.out.println(vc.getDescriptionLocale("cs")); - } - } +// public static void main(String[] args) throws JSONException { +// List cols = new ArrayList(); +// String str = "http://kramerius.lib.cas.cz/search"; +// //http://kramerius.lib.cas.cz/search/api/v5.0/vc +// JSONArray jsonArray = virtualCollectionsFromPoint(appendVCPoint(str)); +// for (int i = 0, ll = jsonArray.length(); i < ll; i++) { +// cols.add(collectionFromJSON(jsonArray.getJSONObject(i))); +// } +// +// for (VirtualCollection vc : cols) { +// System.out.println(vc.getLabel()); +// System.out.println(vc.getPid()); +// System.out.println(vc.getDescriptionLocale("en")); +// System.out.println(vc.getDescriptionLocale("cs")); +// } +// } } From 379144370fb33fcecebcdee5e13a957ec1833702 Mon Sep 17 00:00:00 2001 From: "pavel.stastny@gmail.com" Date: Sun, 15 Mar 2015 14:26:51 +0100 Subject: [PATCH 005/235] Sources client --- .../api/client/v50/client/SourcesClient.java | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/api/client/v50/client/SourcesClient.java diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/client/v50/client/SourcesClient.java b/rest/src/main/java/cz/incad/kramerius/rest/api/client/v50/client/SourcesClient.java new file mode 100644 index 0000000000..86e931f707 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/client/v50/client/SourcesClient.java @@ -0,0 +1,44 @@ +package cz.incad.kramerius.rest.api.client.v50.client; + +import javax.ws.rs.core.MediaType; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.WebResource; + +public class SourcesClient { + + /** + * Informace o zrojich + */ + public static String sources() { + Client c = Client.create(); + WebResource r = c + .resource("http://cdk-test.lib.cas.cz/search/api/v5.0/sources"); + String t = r.accept(MediaType.APPLICATION_JSON).get(String.class); + return t; + } + + public static String source(String src) { + Client c = Client.create(); + WebResource r = c + .resource("http://cdk-test.lib.cas.cz/search/api/v5.0/sources/"+src); + String t = r.accept(MediaType.APPLICATION_JSON).get(String.class); + return t; + } + + + public static void main(String[] args) throws JSONException { + String srcs = sources(); + JSONArray jsonArr = new JSONArray(srcs); + for (int i = 0,ll=jsonArr.length(); i < ll; i++) { + JSONObject jsonO = jsonArr.getJSONObject(i); + String pid = jsonO.getString("pid"); + String url = jsonO.getString("url"); + System.out.println(""+pid +" - "+url); + } + } +} From 85c48194e10158e7c1524bf159a2d8c16d867bb8 Mon Sep 17 00:00:00 2001 From: alberto Date: Tue, 17 Mar 2015 10:59:45 +0100 Subject: [PATCH 006/235] optimized thumbs generations in doc details --- search/web/inc/details/thumbs.jsp | 158 +++++++++++++++++++++--------- search/web/inc/details/tree.jsp | 4 +- 2 files changed, 112 insertions(+), 50 deletions(-) diff --git a/search/web/inc/details/thumbs.jsp b/search/web/inc/details/thumbs.jsp index 002739e3fe..15c69e66ff 100644 --- a/search/web/inc/details/thumbs.jsp +++ b/search/web/inc/details/thumbs.jsp @@ -32,7 +32,7 @@ background:white url(img/loading.gif) no-repeat 50% 50%; } - #tv_container tr td div.t{ + #tv_container div.t{ height:134px; /*width:128px;*/ border:solid white 1px; @@ -43,16 +43,9 @@ -webkit-box-shadow:0 0 6px rgba(0, 0, 0, 0.5); box-shadow:0 0 6px rgba(0, 0, 0, 0.5); position:relative; + display:inline-block; } - #tv_container tr td div.sel{ - border:solid 2px #e75c01; - padding:0; - } - #tv_container tr td div.t>img{ - height:128px; - margin:3px; - } #tv_scroll_table{ padding-left:18px; @@ -70,17 +63,44 @@ line-height: 17px; vertical-align: middle; } + + + #tv_container_row{ + white-space: nowrap; + } + #tv_container_row>div.t>img{ + height:128px; + margin:3px; + } #tv_container_row div.inactive>img{ width:80px; } + #tv_container_row>div.inactive{ + width:80px; + } + + #tv_container.loading{ + background: url('img/loading.gif') center center no-repeat; + cursor: progress; + } + + #tv_container_row>div.sel{ + border:solid 2px #e75c01; + padding:0; + } + div.dost{ + position:absolute;left:2px;top:2px; + } + + #tv_container_row input{ + position:absolute;right:1px;top:1px; + }
-
- - -
+
+
"+ - ""; - } - - - public String getLevelsModelSelectionArray() { - //StringTemplate template = new StringTemp - StringTemplateGroup grp = stGroup(); - StringTemplate st = grp.getInstanceOf("levelModelsSelection"); - HashMap model = new HashMap(); { - HttpServletRequest request = this.requestProvider.get(); - String pidPath = request.getParameter("pid_path"); - String path = request.getParameter("path"); - if ((pidPath != null) && (path !=null)) { - String[] uuids = pidPath.split("/"); - String[] models = path.split("/"); - for (int i = 0; i < models.length; i++) { - String key = (i+1)+"_"+models[i]; - model.put(key, uuids[i]); - } - } - } - st.setAttribute("data", model); - return st.toString(); - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/ItemViewObject.java b/search/src/java/cz/incad/Kramerius/views/ItemViewObject.java deleted file mode 100644 index e7fc2f0e57..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/ItemViewObject.java +++ /dev/null @@ -1,45 +0,0 @@ -package cz.incad.Kramerius.views; - -import javax.inject.Named; -import javax.servlet.ServletContext; -import javax.servlet.http.HttpServletRequest; - -import com.google.inject.Inject; - -import cz.incad.Kramerius.exts.menu.context.ContextMenu; -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.MostDesirable; - -public class ItemViewObject { - - public static final java.util.logging.Logger LOGGER = java.util.logging.Logger - .getLogger(ItemViewObject.class.getName()); - - @Inject - ServletContext servletContext; - - @Inject - MostDesirable mostDesirable; - - @Inject - HttpServletRequest request; - - @Inject - @Named("securedFedoraAccess") - FedoraAccess fedoraAccess; - - public ItemViewObject() { - super(); - } - - /** - * Only save to most desirable access - * - * @return - */ - public String getMostDesirableAccess() { - mostDesirable.saveAccess(request.getParameter("pid"), new java.util.Date()); - return ""; - } - -} diff --git a/search/src/java/cz/incad/Kramerius/views/OutputTemplateViewObjectItem.java b/search/src/java/cz/incad/Kramerius/views/OutputTemplateViewObjectItem.java deleted file mode 100644 index beea7507f8..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/OutputTemplateViewObjectItem.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright (C) 2012 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views; - -import java.io.IOException; -import java.io.StringWriter; -import java.util.logging.Level; - -import cz.incad.kramerius.processes.LRProcess; -import cz.incad.kramerius.processes.LRProcessDefinition; -import cz.incad.kramerius.processes.template.OutputTemplateFactory; -import cz.incad.kramerius.processes.template.ProcessOutputTemplate; - -public class OutputTemplateViewObjectItem { - - static java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(OutputTemplateViewObjectItem.class.getName()); - - private String clz; - private LRProcess lrProcess; - private LRProcessDefinition lrProcessDefinition; - private ProcessOutputTemplate outTemplate; - - - public OutputTemplateViewObjectItem(String clz, OutputTemplateFactory factory, LRProcess lrProcess, LRProcessDefinition definition) throws ClassNotFoundException, InstantiationException, IllegalAccessException { - super(); - this.clz = clz; - this.outTemplate = factory.create(this.clz); - this.lrProcess = lrProcess; - this.lrProcessDefinition = definition; - } - - - public String getId() { - return this.outTemplate.getOutputTemplateId(); - } - - public String getName() { - try { - StringWriter stWriter = new StringWriter(); - outTemplate.renderName(this.lrProcess, this.lrProcessDefinition, stWriter); - return stWriter.toString(); - } catch (IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - return ""; - } - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/PdfGenerateViewObject.java b/search/src/java/cz/incad/Kramerius/views/PdfGenerateViewObject.java deleted file mode 100644 index eed2c567d1..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/PdfGenerateViewObject.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views; - -import java.io.IOException; -import java.util.ResourceBundle; - -import javax.xml.parsers.ParserConfigurationException; - -import org.xml.sax.SAXException; - -import antlr.RecognitionException; -import antlr.TokenStreamException; -import cz.incad.kramerius.utils.conf.KConfiguration; - -public class PdfGenerateViewObject extends AbstractPrintViewObject { - - - public String getNumberOfGeneratedPages() throws IOException, RecognitionException, TokenStreamException, ParserConfigurationException, SAXException { - ResourceBundle bundle = this.resourceBundleService.getResourceBundle("labels", this.locale); - return bundle.getString("pdf.numberOfPages"); - } - - public String getMaxNumberOfPages() { - return ""+KConfiguration.getInstance().getConfiguration().getInt("generatePdfMaxRange"); - } - - -} diff --git a/search/src/java/cz/incad/Kramerius/views/PreparedForProfileView.java b/search/src/java/cz/incad/Kramerius/views/PreparedForProfileView.java deleted file mode 100644 index 35a44c8bc0..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/PreparedForProfileView.java +++ /dev/null @@ -1,159 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.ResourceBundle; -import java.util.Set; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpSession; - - -import com.google.inject.Inject; -import com.google.inject.Provider; - -import cz.incad.Kramerius.users.ProfilePrepareUtils; -import cz.incad.kramerius.service.ResourceBundleService; - -public class PreparedForProfileView { - - //"results":{"sorting_dir":"desc","columns":"2","sorting":"title"}, - //"client_locale":"cs"}" - - protected List KEYS = new ArrayList(Arrays.asList(new String[]{"sorting","sorting_dir","columns","client_locale"})); - protected HashMap DEFAULT_VALS = new HashMap(); { - DEFAULT_VALS.put("sorting","title"); - DEFAULT_VALS.put("sorting_dir","desc"); - DEFAULT_VALS.put("columns","2"); - }; - - - @Inject - Provider requestProvider; - - @Inject - Provider localesProvider; - - @Inject - ResourceBundleService resourceBundleService; - - - - - public PreparedForProfileView() { - super(); - - } - - - public List getProfileCandidateItems() throws IOException { - - Locale locale = this.localesProvider.get(); - ResourceBundle bundle = this.resourceBundleService.getResourceBundle("labels", locale); - - this.DEFAULT_VALS.put("client_locale", locale.getLanguage()); - - HttpSession session = this.requestProvider.get().getSession(); - Map preparedProperties = ProfilePrepareUtils.getPreparedProperties(session); - if (preparedProperties == null) preparedProperties = new HashMap(); - - List items = new ArrayList(); - for (String key : KEYS) { - boolean flag = preparedProperties.containsKey(key); - String value = flag ? preparedProperties.get(key) : DEFAULT_VALS.get(key); - ProfileCandidateItem item = item(bundle, key, value); - item.setChecked(flag); - items.add(item); - } - return items; - } - - - public ProfileCandidateItem item(ResourceBundle bundle, String key, String value) { - ProfileCandidateItem candidateItem = new ProfileCandidateItem(key, value); - - if (bundle.containsKey("userprofile.forsave.value."+value)) { - candidateItem.setLocalizedValue(bundle.getString("userprofile.forsave.value."+value)); - } - if (bundle.containsKey("userprofile.forsave."+key)) { - candidateItem.setLocalizedKey(bundle.getString("userprofile.forsave."+key)); - } - - return candidateItem; - } - - - public static class ProfileCandidateItem { - - private String key; - private String value; - - private String localizedKey; - private String localizedValue; - - private boolean checked = false; - - - private ProfileCandidateItem(String key, String value) { - super(); - this.key = key; - this.value = value; - } - - public String getKey() { - return key; - } - - public String getValue() { - return value; - } - - public void setLocalizedKey(String localizedKey) { - this.localizedKey = localizedKey; - } - - public String getLocalizedKey() { - if (this.localizedKey == null) return this.key; - return localizedKey; - } - - public void setLocalizedValue(String localizedValue) { - this.localizedValue = localizedValue; - } - public String getLocalizedValue() { - if (this.localizedValue == null) return this.value; - return localizedValue; - } - - public boolean isChecked() { - return checked; - } - - public void setChecked(boolean checked) { - this.checked = checked; - } - - } - -} diff --git a/search/src/java/cz/incad/Kramerius/views/PrintViewObject.java b/search/src/java/cz/incad/Kramerius/views/PrintViewObject.java deleted file mode 100644 index becee89873..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/PrintViewObject.java +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views; - -import cz.incad.kramerius.Initializable; - -public class PrintViewObject extends AbstractPrintViewObject { - - - -} diff --git a/search/src/java/cz/incad/Kramerius/views/ProcessLogsViewObject.java b/search/src/java/cz/incad/Kramerius/views/ProcessLogsViewObject.java deleted file mode 100644 index 6f01f542ba..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/ProcessLogsViewObject.java +++ /dev/null @@ -1,192 +0,0 @@ -package cz.incad.Kramerius.views; - -import cz.incad.kramerius.processes.LRDefinitionAction; -import cz.incad.kramerius.processes.LRProcess; -import cz.incad.kramerius.processes.LRProcessDefinition; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.util.*; -import java.util.logging.Level; - -public class ProcessLogsViewObject { - - public static final int BUFFER_SIZE = 1 << 9; - - public static final java.util.logging.Logger LOGGER = java.util.logging.Logger - .getLogger(ProcessLogsViewObject.class.getName()); - - private String stdFrom; - private String errFrom; - - - private String count; - private LRProcess process; - private LRProcessDefinition definition; - private ResourceBundle resBundle; - - public ProcessLogsViewObject(String stdFrom, String errFrom, String count, LRProcess process, LRProcessDefinition definition, ResourceBundle re) { - super(); - this.stdFrom = stdFrom != null ? stdFrom : "0"; - this.errFrom = errFrom != null ? errFrom : "0"; - - - this.count = count != null ? count : "4096"; - this.process = process; - this.definition = definition; - this.resBundle = re; - } - - public String getProcessUUID() { - return process.getUUID(); - } - - public File getProcessWorkingDirectory() { - return process.processWorkingDirectory(); - } - - public File getStdOutDirectory() { - return new File(getProcessWorkingDirectory().getAbsolutePath()+File.separator+ this.definition.getStandardStreamFolder()+File.separator+"stout.out"); - } - - public File getErrOutDirectory() { - return new File(getProcessWorkingDirectory().getAbsolutePath()+File.separator+ this.definition.getErrStreamFolder()+File.separator+"sterr.err"); - } - - public long getErrorFileSize() throws IOException { - RandomAccessFile errorProcessRAFile = null; - try { - errorProcessRAFile = this.process.getErrorProcessRAFile(); - return errorProcessRAFile.length(); - } catch (IOException ex) { - LOGGER.log(Level.FINE,ex.getMessage(),ex); - return 0; - } finally { - if (errorProcessRAFile != null) errorProcessRAFile.close(); - } - } - - public long getStdFileSize() throws IOException { - RandomAccessFile stdProcessRAFile = null; - try { - stdProcessRAFile = this.process.getStandardProcessRAFile(); - return stdProcessRAFile.length(); - } catch (IOException ex) { - LOGGER.log(Level.FINE,ex.getMessage(),ex); - return 0; - } finally { - if (stdProcessRAFile != null) stdProcessRAFile.close(); - } - } - - public String getErrOutData() { - RandomAccessFile raf = null; - try { - raf = this.process.getErrorProcessRAFile(); - return bufferFromRAF(raf, this.errFrom); - } catch (FileNotFoundException e) { - LOGGER.log(Level.FINE, e.getMessage(), e); - } catch (IOException e) { - LOGGER.log(Level.FINE, e.getMessage(), e); - } finally { - try { - if (raf != null) raf.close(); - } catch (IOException e) { - LOGGER.log(Level.FINE, e.getMessage(), e); - } - } - return ""; - } - - - - public String getStdOutData() { - RandomAccessFile raf = null; - try { - raf = this.process.getStandardProcessRAFile(); - return bufferFromRAF(raf, this.stdFrom); - } catch (FileNotFoundException e) { - LOGGER.log(Level.FINE, e.getMessage(), e); - } catch (IOException e) { - LOGGER.log(Level.FINE, e.getMessage(), e); - } - return ""; - } - - - - - - public boolean isActionsDefined() { - return !this.definition.getActions().isEmpty(); - } - - public List> getActionURLs() { - List> lst = new ArrayList>(); - List actions = this.definition.getActions(); - for (LRDefinitionAction defAct : actions) { - Map map = new HashMap(); - map.put("name", defAct.getName()); - map.put("url", defAct.getActionURL()); - map.put("i18nName", this.resBundle.getString(defAct.getResourceBundleKey())); - lst.add(map); - } - return lst; - } - - - private String bufferFromRAF(RandomAccessFile raf, String from) throws IOException { - long fromL = Long.parseLong(from); - long countL = Long.parseLong(this.count); - - byte[] buffer = new byte[(int) countL]; - raf.seek(fromL); - int read = raf.read(buffer); - if (read >= 0) { - byte[] nbuffer = new byte[read]; - System.arraycopy(buffer, 0, nbuffer, 0, read); - return new String(nbuffer); - } else return ""; - } - - - public String getNextStandardOutputAHREF() { - int fromI = Integer.parseInt(this.stdFrom); - int countI = Integer.parseInt(this.count); - int next = fromI + countI; - return " next > "; - } - - - public String getNextErrorOutputAHREF() { - int fromI = Integer.parseInt(this.errFrom); - int countI = Integer.parseInt(this.count); - int next = fromI + countI; - return " next > "; - } - - public String getPrevStandardOutputAHREF() { - int fromI = Integer.parseInt(this.stdFrom); - int countI = Integer.parseInt(this.count); - int next = fromI - countI; - return " < prev "; - } - - - public String getPrevErrorOutputAHREF() { - int fromI = Integer.parseInt(this.errFrom); - int countI = Integer.parseInt(this.count); - int next = fromI - countI; - return " < prev "; - } - - - - public static void main(String[] args) { - System.out.println(BUFFER_SIZE); - } - - -} diff --git a/search/src/java/cz/incad/Kramerius/views/ProcessStateWrapper.java b/search/src/java/cz/incad/Kramerius/views/ProcessStateWrapper.java deleted file mode 100644 index 4a70edb75c..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/ProcessStateWrapper.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views; - -import java.util.ArrayList; -import java.util.List; - -import cz.incad.kramerius.processes.States; - -public class ProcessStateWrapper { - - private States state; - private boolean selected = false; - - public ProcessStateWrapper(States state) { - super(); - this.state = state; - } - - public String getName() { - return (this.state != null ? state.name() : "All"); - } - - public int getVal() { - return (this.state != null ? state.getVal() : -1); - } - - public void setSelected(boolean selected) { - this.selected = selected; - } - - public boolean isSelected() { - return selected; - } - - public static List wrap(boolean withall, States... states) { - List wrappers = new ArrayList(); - if (withall) { - wrappers.add(new ProcessStateWrapper(null)); - } - for (States st : states) { - wrappers.add(new ProcessStateWrapper(st)); - } - return wrappers; - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/ProcessViewObject.java b/search/src/java/cz/incad/Kramerius/views/ProcessViewObject.java deleted file mode 100644 index 15682247d0..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/ProcessViewObject.java +++ /dev/null @@ -1,388 +0,0 @@ -package cz.incad.Kramerius.views; - -import java.io.IOException; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Calendar; -import java.util.Date; -import java.util.List; -import java.util.Locale; -import java.util.ResourceBundle; -import java.util.logging.Level; - -import org.antlr.stringtemplate.StringTemplate; - -import com.google.inject.Provider; - -import cz.incad.kramerius.processes.BatchStates; -import cz.incad.kramerius.processes.LRDefinitionAction; -import cz.incad.kramerius.processes.LRProcess; -import cz.incad.kramerius.processes.LRProcessDefinition; -import cz.incad.kramerius.processes.LRProcessOrdering; -import cz.incad.kramerius.processes.States; -import cz.incad.kramerius.processes.template.OutputTemplateFactory; -import cz.incad.kramerius.processes.template.ProcessOutputTemplate; -import cz.incad.kramerius.security.User; -import cz.incad.kramerius.security.database.TypeOfOrdering; -import cz.incad.kramerius.service.ResourceBundleService; -import cz.incad.kramerius.utils.database.Offset; - -public class ProcessViewObject { - - public static final java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(ProcessViewObject.class.getName()); - - public static final SimpleDateFormat FORMAT = new SimpleDateFormat("yyyy/dd/MM - HH:mm:ss"); - - private LRProcess lrProcess; - private LRProcessDefinition definition; - private LRProcessOrdering ordering; - private cz.incad.kramerius.utils.database.Offset offset; - private TypeOfOrdering typeOfOrdering; - //private String lrUrl; - private ResourceBundleService bundleService; - private Locale locale; - - private OutputTemplateFactory outputTemplateFactory; - - private List childProcesses = new ArrayList(); - - private String page; - - public ProcessViewObject(LRProcess lrProcess, LRProcessDefinition definition, LRProcessOrdering ordering, Offset offset, TypeOfOrdering typeOfOrdering, ResourceBundleService service, Locale locale, OutputTemplateFactory factory, String page) { - super(); - this.lrProcess = lrProcess; - this.ordering = ordering; - this.offset = offset; - this.typeOfOrdering = typeOfOrdering; - this.definition = definition; - this.bundleService = service; - this.locale = locale; - this.outputTemplateFactory = factory; - this.page = page; - } - - public String getPid() { - return lrProcess.getPid(); - } - - public String getName() { - try { - String unnamed = bundleService.getResourceBundle("labels", locale).getString("administrator.processes.unnamedprocess"); - if (lrProcess.getProcessName() == null) { - return unnamed + "
" + lrProcess.getDescription() + ""; - } else { - return lrProcess.getProcessName() + "
" + lrProcess.getDescription() + ""; - } - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return ""; - } - } - - public String getFormatedProcessName() { - return getName(); - // if (this.definition.getProcessOutputURL() != null) { - // return - // ""+getName()+""; - // } else { - // return - // ""+getName()+""; - // } - } - - public String getSimpleProcessName() { - try { - String unnamed = bundleService.getResourceBundle("labels", locale).getString("administrator.processes.unnamedprocess"); - return this.lrProcess.getProcessName() != null ? lrProcess.getProcessName() : unnamed; - } catch (IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - return ""; - } - } - - public String getProcessState() { - return lrProcess.getProcessState().name(); - } - - - public boolean isRunningState() { - return lrProcess.getProcessState().equals(States.RUNNING); - } - - public boolean isWarningState() { - return lrProcess.getProcessState().equals(States.WARNING); - } - - public boolean isFailedState() { - return lrProcess.getProcessState().equals(States.FAILED); - } - - - public boolean isFailedBatchState() { - return lrProcess.getBatchState() != null && lrProcess.getBatchState().equals(BatchStates.BATCH_FAILED); - } - - public boolean isWarningBatchState() { - return lrProcess.getBatchState() != null && lrProcess.getBatchState().equals(BatchStates.BATCH_WARNING); - } - - public boolean isRunningdBatchState() { - return lrProcess.getBatchState() != null && lrProcess.getBatchState().equals(BatchStates.BATCH_STARTED); - } - - public String getBatchState() { - BatchStates bState = lrProcess.getBatchState(); - if (!bState.equals(BatchStates.NO_BATCH)) { - return lrProcess.getBatchState().name(); - } else return ""; - } - - public String getFinished() { - Date date = new Date(lrProcess.getFinishedTime()); - if (date.getTime() != 0) { - return FORMAT.format(date); - } else return ""; - } - - public String getDuration() throws IOException { - if (lrProcess.getFinishedTime() != 0) { - long startTime = lrProcess.getStartTime(); - long finishTime = lrProcess.getFinishedTime(); - - ResourceBundle bundle = bundleService.getResourceBundle("labels", locale); - StringBuilder builder = formatDuration(startTime, finishTime, bundle); - - return builder.toString(); - - } else return ""; - } - - public static StringBuilder formatDuration(long startTime, long finishTime, ResourceBundle bundle) { - int days = 0; - int hours = 0; - int minutes = 0; - int seconds = 0; - long milisconds = 0; - - Calendar startTimeCal = Calendar.getInstance(); startTimeCal.setTimeInMillis(startTime); - Calendar finishTimeCal = Calendar.getInstance(); finishTimeCal.setTimeInMillis(finishTime); - - Calendar processingCal = Calendar.getInstance(); - processingCal.setTimeInMillis(startTime); - - //final long day = 1000*60*60*24; - if (moreThenPeriod(finishTimeCal.getTimeInMillis(), processingCal.getTimeInMillis(), 1000*60*60*24)) { - days = duration(finishTimeCal, processingCal, 1000*60*60*24); - } - - if (moreThenPeriod(finishTimeCal.getTimeInMillis(),processingCal.getTimeInMillis(),1000*60*60)) { - hours = duration(finishTimeCal, processingCal, 1000*60*60); - } - - if (moreThenPeriod(finishTimeCal.getTimeInMillis(),processingCal.getTimeInMillis(), 1000*60)) { - minutes = duration(finishTimeCal, processingCal, 1000*60); - } - - if (moreThenPeriod(finishTimeCal.getTimeInMillis(),processingCal.getTimeInMillis(), 1000)) { - seconds = duration(finishTimeCal, processingCal, 1000); - } - - milisconds = finishTimeCal.getTimeInMillis()-processingCal.getTimeInMillis(); - - StringBuilder builder = new StringBuilder(); - - - if (days > 0) builder.append(days).append(" ").append(bundle.getString("administrator.processes.duration.days")).append(" "); - if (hours > 0) builder.append(hours).append(" ").append(bundle.getString("administrator.processes.duration.hours")).append(" "); - if (minutes > 0) builder.append(minutes).append(" ").append(bundle.getString("administrator.processes.duration.minutes")).append(" "); - if (seconds > 0) builder.append(seconds).append(" ").append(bundle.getString("administrator.processes.duration.seconds")).append(" "); - if (milisconds > 0) builder.append(milisconds).append(" ").append(bundle.getString("administrator.processes.duration.miliseconds")).append(" "); - return builder; - } - - - private static int duration(Calendar finishTime, Calendar startTime, long period) { - int calcualated = 0; - - long ftime = finishTime.getTimeInMillis(); - long stime = startTime.getTimeInMillis(); - - while((stime+period) <= ftime) { - stime += period; - calcualated +=1; - } - - startTime.setTimeInMillis(stime); - return calcualated; - - } - - private static boolean moreThenPeriod(long finishTime, long startTime, long period) { - return (finishTime - startTime) >= period; - } - - - - - public String getStart() { - Date date = new Date(lrProcess.getStartTime()); - if (date.getTime() != 0) { - return FORMAT.format(date); - } else return ""; - } - - public String getPlanned() { - Date date = new Date(lrProcess.getPlannedTime()); - if (date.getTime() != 0L) { - return FORMAT.format(date); - } else return ""; - } - - public boolean isMasterProcess() { - return lrProcess.isMasterProcess(); - } - - public String getUUID() { - return lrProcess.getUUID(); - } - - public String getTreeIcon() { - if (lrProcess.isMasterProcess()) { - return ""; - } else { - return ""; - } - } - - // function killAndRefresh(url,ordering, offset, size, type) { - - public String getKillURL() { - - - try { - - if ((this.lrProcess.getProcessState().equals(States.RUNNING)) || (this.lrProcess.getProcessState().equals(States.PLANNED))) { - String url = "lr?action=stop&uuid=" + this.lrProcess.getUUID(); - String fn = "showConfirmDialog(dictionary['administrator.processes.kill.process.confirm'], function() {" + - "processes.doActionAndRefresh('" + url + "','" + this.ordering.name() + "'," + (this.page != null ? ("'"+this.page+"'") :null) + "," + this.offset.getSize() + ",'" + this.typeOfOrdering.name() + "');" + - "});"; - - String renderedAHREF = "" - + bundleService.getResourceBundle("labels", locale).getString("administrator.processes.kill.process") + ""; - return renderedAHREF; - - } else { - return ""+bundleService.getResourceBundle("labels", locale).getString("administrator.processes.kill.process")+"" ; - } - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return ""; - } - } - - - public String getDeleteURL() { - try { - - boolean notRunning = States.notRunningState(this.lrProcess.getProcessState()); - boolean notInBatch = BatchStates.notRunningBatch(this.lrProcess.getBatchState()); - - - if (notRunning && notInBatch) { - - String url = "lr?action=delete&uuid=" + this.lrProcess.getUUID(); - String fn = "showConfirmDialog(dictionary['administrator.processes.delete.process.confirm'], function() {" + - "processes.doActionAndRefresh('" + url + "','" + this.ordering.name() + "'," + (this.page != null ? ("'"+this.page+"'") : null ) + "," + this.offset.getSize() + ",'" + this.typeOfOrdering.name() + "');" + - "});"; - - String renderedAHREF = "" - + bundleService.getResourceBundle("labels", locale).getString("administrator.processes.delete.process") + ""; - return renderedAHREF; - } else { - return ""+bundleService.getResourceBundle("labels", locale).getString("administrator.processes.delete.process")+""; - } - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return ""; - } - } - - public String getLogsURLs() { - return getActionAHREF(this.definition.getLogsAction()); - } - - public String[] getActionsURLs() { - List hrefs = new ArrayList(); - List actions = this.definition.getActions(); - for (int i = 0, ll = actions.size(); i < ll; i++) { - hrefs.add(getActionAHREF(actions.get(i))); - } - return (String[]) hrefs.toArray(new String[hrefs.size()]); - } - - - public String getStartedBy() { - StringBuilder builder = new StringBuilder(); - String firstName = lrProcess.getFirstname(); - String surName = lrProcess.getSurname(); - String loginname = lrProcess.getLoginname(); - builder.append(loginname).append(" (").append(firstName).append(" ").append(surName).append(")"); - return builder.toString(); - } - - - private String getActionAHREF(LRDefinitionAction act) { - try { - String bundleKey = act.getResourceBundleKey(); - if (bundleKey != null) { - return "" + bundleService.getResourceBundle("labels", locale).getString(bundleKey) + ""; - } else { - LOGGER.info(" action '" + act.getName() + "' has no bundle key"); - return "" + act.getName() + ""; - } - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return ""; - } - } - - public void addChildProcess(ProcessViewObject pw) { - this.childProcesses.add(pw); - } - - public void removeChildProcess(ProcessViewObject pw) { - this.childProcesses.remove(pw); - } - - public List getChildProcesses() { - return this.childProcesses; - } - - - public boolean isOutputTemplatesDefined() { - return this.definition.isOutputTemplatesDefined(); - } - - - public List getOutputTemplateViewObjects() { - try { - List outItems = new ArrayList(); - List outputTemplateClasses = this.definition.getOutputTemplateClasses(); - for (String clzName : outputTemplateClasses) { - outItems.add(new OutputTemplateViewObjectItem(clzName, this.outputTemplateFactory, this.lrProcess, this.definition)); - } - return outItems; - } catch (ClassNotFoundException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } catch (InstantiationException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } catch (IllegalAccessException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } - return new ArrayList(); - } - - - -} diff --git a/search/src/java/cz/incad/Kramerius/views/ProcessesViewObject.java b/search/src/java/cz/incad/Kramerius/views/ProcessesViewObject.java deleted file mode 100644 index ecc9271059..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/ProcessesViewObject.java +++ /dev/null @@ -1,806 +0,0 @@ -package cz.incad.Kramerius.views; - -import java.io.IOException; -import java.io.StringReader; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.logging.Level; - -import javax.servlet.http.HttpServletRequest; - -import com.google.inject.Inject; -import com.google.inject.Provider; - -import antlr.RecognitionException; -import antlr.TokenStreamException; -import cz.incad.kramerius.Initializable; -import cz.incad.kramerius.processes.BatchStates; -import cz.incad.kramerius.processes.DefinitionManager; -import cz.incad.kramerius.processes.LRProcess; -import cz.incad.kramerius.processes.LRProcessDefinition; -import cz.incad.kramerius.processes.LRProcessManager; -import cz.incad.kramerius.processes.LRProcessOrdering; -import cz.incad.kramerius.processes.States; -import cz.incad.kramerius.processes.template.OutputTemplateFactory; -import cz.incad.kramerius.security.database.TypeOfOrdering; -import cz.incad.kramerius.service.ResourceBundleService; -import cz.incad.kramerius.utils.database.Offset; -import cz.incad.kramerius.utils.database.SQLFilter; -import cz.incad.kramerius.utils.database.SQLFilter.Tripple; -import cz.incad.kramerius.utils.database.SQLFilter.TypesMapping; -import cz.incad.kramerius.utils.params.ParamsLexer; -import cz.incad.kramerius.utils.params.ParamsParser; - -public class ProcessesViewObject implements Initializable { - - private static final int SMALL_SET_OF_DIRECT_PAGES = 25; - - private static final int LARGE_SET_OF_DIRECT_PAGES = 25; - - public static final java.util.logging.Logger LOGGER = java.util.logging.Logger - .getLogger(ProcessesViewObject.class.getName()); - - @Inject - protected LRProcessManager processManager; - - @Inject - protected DefinitionManager definitionManager; - - @Inject - protected ResourceBundleService bundleService; - - @Inject - protected Provider localesProvider; - - @Inject - protected Provider requestProvider; - - @Inject - protected OutputTemplateFactory outputTemplateFactory; - - private LRProcessOrdering ordering; - // private LRProcessOffset offset; - private TypeOfOrdering typeOfOrdering; - - private String page; - private String pageSize; - - private String lrUrl; - - private SQLFilter filter; - - private String filterParam; - - private int numberOfRunningProcesses = -1; - - public ProcessesViewObject() throws RecognitionException { - super(); - } - - public void init() { - try { - - String type = this.requestProvider.get().getParameter("type"); - if ((type == null) || (type.trim().equals(""))) { - type = "DESC"; - } - this.typeOfOrdering = TypeOfOrdering.valueOf(type); - - String ordering = this.requestProvider.get().getParameter( - "ordering"); - if ((ordering == null) || (ordering.trim().equals(""))) { - ordering = LRProcessOrdering.PLANNED.name(); - } - this.ordering = LRProcessOrdering.valueOf(ordering); - - this.filterParam = this.requestProvider.get() - .getParameter("filter"); - this.filter = this.createProcessFilter(); - - String size = this.requestProvider.get().getParameter("size"); - if ((size == null) || (size.trim().equals(""))) { - size = "5"; - } - this.pageSize = size; - - String page = this.requestProvider.get().getParameter("page"); - if (page != null) { - this.page = page; - } else { - this.page = "" + getFirstPage(); - } - - } catch (RecognitionException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - throw new RuntimeException(e); - } - } - - public SQLFilter getFilter() { - return filter; - } - - public List getProcesses() { - int pageSize = getDefaultPageSize(); - if (this.isCurrentFirstPage()) { - pageSize = pageSize + (getNumberOfRunningProcess() % getDefaultPageSize()); - } - - Offset offset = new Offset("" + getOffset(getPage()), "" + pageSize); - List lrProcesses = this.processManager - .getLongRunningProcessesAsGrouped(this.ordering, - this.typeOfOrdering, offset, this.filter); - List objects = new ArrayList(); - for (LRProcess lrProcess : lrProcesses) { - LRProcessDefinition def = this.definitionManager - .getLongRunningProcessDefinition(lrProcess - .getDefinitionId()); - ProcessViewObject pw = new ProcessViewObject(lrProcess, def, - this.ordering, offset, this.typeOfOrdering, - this.bundleService, this.localesProvider.get(), - this.outputTemplateFactory, this.page); - if (lrProcess.isMasterProcess()) { - List childSubprecesses = this.processManager - .getLongRunningProcessesByGroupToken(lrProcess - .getGroupToken()); - for (LRProcess child : childSubprecesses) { - if (!child.getUUID().equals(lrProcess.getUUID())) { - LRProcessDefinition childDef = this.definitionManager - .getLongRunningProcessDefinition(child - .getDefinitionId()); - ProcessViewObject childPW = new ProcessViewObject( - child, childDef, this.ordering, offset, - this.typeOfOrdering, this.bundleService, - this.localesProvider.get(), - this.outputTemplateFactory, this.page); - pw.addChildProcess(childPW); - } - } - } - objects.add(pw); - } - return objects; - } - - private int getOffset(int page) { - // max page - min offset - int offsetPage = Math.max(getNumberOfPages() - 1 - page, 0); - if (offsetPage >= 1) { - // more then one -> must have bigger offset - return offsetPage * getDefaultPageSize() - + (getNumberOfRunningProcess() % getDefaultPageSize()); - } else { - return offsetPage * getDefaultPageSize(); - } - } - - private SQLFilter createProcessFilter() throws RecognitionException { - if (this.filterParam == null) - return null; - try { - ParamsParser paramsParser = new ParamsParser(new ParamsLexer( - new StringReader(this.filterParam))); - List params = paramsParser.params(); - List tripples = new ArrayList(); - for (Object object : params) { - List trippleList = (List) object; - Tripple tripple = createTripple(trippleList); - if (tripple.getVal() != null) { - tripples.add(tripple); - } - } - - TypesMapping types = new TypesMapping(); - types.map("status", new SQLFilter.IntegerConverter()); - types.map("batch_status", new SQLFilter.IntegerConverter()); - types.map("planned", new SQLFilter.DateConvereter()); - types.map("started", new SQLFilter.DateConvereter()); - types.map("finished", new SQLFilter.DateConvereter()); - - SQLFilter filter = SQLFilter.createFilter(types, tripples); - // TODO: do it better - if (filter != null) { - - Tripple statusTripple = filter.findTripple("status"); - if (statusTripple != null) { - if (((Integer) statusTripple.getVal()) == -1) { - filter.removeTripple(statusTripple); - } - } - - Tripple bstatusTripple = filter.findTripple("batch_status"); - if (bstatusTripple != null) { - if (((Integer) bstatusTripple.getVal()) == -1) { - filter.removeTripple(bstatusTripple); - } - } - - } - - return filter; - } catch (TokenStreamException te) { - te.printStackTrace(); - return null; - } catch (UnsupportedEncodingException e) { - e.printStackTrace(); - return null; - } - } - - private Tripple createTripple(List trpList) throws UnsupportedEncodingException { - if (trpList.size() == 3) { - String name = (String) trpList.get(0); - String op = (String) trpList.get(1); - String val = (String) trpList.get(2); - // decode if name - if (name.equals("name")) { - val = URLDecoder.decode(val, "UTF-8"); - } - Tripple trp = new Tripple(name, val, op); - return trp; - } else - return null; - } - - public boolean getHasPrevious() { - int page = getPage(); - int numberOfPages = getNumberOfPages(); - return page < (numberOfPages - 1); - } - - public boolean getHasNext() { - return getPage() > 0; - } - - public int getNumberOfRunningProcess() { - if (this.numberOfRunningProcesses == -1) { - this.numberOfRunningProcesses = this.processManager - .getNumberOfLongRunningProcesses(this.filter); - } - return this.numberOfRunningProcesses; - } - - public int getDefaultPageSize() { - return Integer.parseInt(this.pageSize); - } - public int getPageSize() { - return Integer.parseInt(this.pageSize); - } - - public int getNumberOfPages() { - return getNumberOfRunningProcess() / getDefaultPageSize(); - } - - public int getPage() { - return Integer.parseInt(this.page); - } - - public String getPageLabel() { - return page; - } - - public int getPrevPageValue() { - return (getPage() < getNumberOfPages()) ? getPage() + 1 : 0; - } - - public int getNextPageValue() { - return (getPage() > 0) ? getPage() - 1 : getNumberOfPages(); - } - - public int getFirstPage() { - return Math.max(getNumberOfPages() - 1, 0); - } - - public int getLastPage() { - return 0; - } - - public String getOrdering() { - return this.ordering.toString(); - } - - public String getTypeOfOrdering() { - return this.typeOfOrdering.getTypeOfOrdering(); - } - - public boolean isNecessaryDisplayMorePages() { - return this.getNumberOfPages() > SMALL_SET_OF_DIRECT_PAGES; - } - - public List getLargeSetOfDirectPates() { - List hrefs = new ArrayList(); - int pages = getNumberOfPages(); - int page = getPage(); - for (int i = pages - 1; i >= 0; i--) { - String href = " " + i - + ""; - hrefs.add(href); - } - return hrefs; - - } - - public List getSmallSetOfDirectPages() { - List hrefs = new ArrayList(); - int pageFrom = Math.min(Math.max(getPage(), SMALL_SET_OF_DIRECT_PAGES), - getNumberOfPages() - 1); - int pageTo = Math.max(pageFrom - SMALL_SET_OF_DIRECT_PAGES, 0); - for (int i = pageFrom; i >= pageTo; i--) { - String href = " " - + i - + ""; - hrefs.add(href); - } - - return hrefs; - - } - - public List getDirectPages() { - List hrefs = new ArrayList(); - int pages = getNumberOfPages(); - for (int i = pages - 1; i >= 0; i--) { - String href = " " - + i - + ""; - hrefs.add(href); - } - return hrefs; - } - - public String getNextPageAHREF() { - try { - String nextString = bundleService.getResourceBundle("labels", - this.localesProvider.get()).getString( - "administrator.processes.next"); - return " " - + nextString - + " "; - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return "\"next\""; - } - } - - public String getPrevPageAHREF() { - try { - String prevString = bundleService.getResourceBundle("labels", - this.localesProvider.get()).getString( - "administrator.processes.prev"); - return " " - + prevString + " "; - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return "\"prev\" "; - } - } - - private TypeOfOrdering switchOrdering() { - return this.typeOfOrdering.equals(TypeOfOrdering.ASC) ? TypeOfOrdering.DESC - : TypeOfOrdering.ASC; - } - - public String getOrderingIcon() { - return this.typeOfOrdering.equals(TypeOfOrdering.ASC) ? "" - : ""; - } - - public boolean isNameOrdered() { - return this.ordering.equals(LRProcessOrdering.NAME); - } - - public String getNameOrdering() { - try { - String nameString = bundleService.getResourceBundle("labels", - this.localesProvider.get()).getString( - "administrator.processes.name"); - LRProcessOrdering nOrdering = LRProcessOrdering.NAME; - boolean changeTypeOfOrdering = this.ordering.equals(nOrdering); - return newOrderingURL(nOrdering, nameString, - changeTypeOfOrdering ? switchOrdering() - : TypeOfOrdering.ASC); - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return e.getMessage(); - } - } - - public boolean isStartedDateOrdered() { - return this.ordering.equals(LRProcessOrdering.STARTED); - } - - public String getDateOrdering() { - try { - String startedString = bundleService.getResourceBundle("labels", - this.localesProvider.get()).getString( - "administrator.processes.started"); - LRProcessOrdering nOrdering = LRProcessOrdering.STARTED; - boolean changeTypeOfOrdering = this.ordering.equals(nOrdering); - return newOrderingURL(nOrdering, startedString, - changeTypeOfOrdering ? switchOrdering() - : TypeOfOrdering.ASC); - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return e.getMessage(); - } - } - - public boolean isPlannedDateOrdered() { - return this.ordering.equals(LRProcessOrdering.PLANNED); - } - - public String getPlannedDateOrdering() { - try { - String startedString = bundleService.getResourceBundle("labels", - this.localesProvider.get()).getString( - "administrator.processes.planned"); - LRProcessOrdering nOrdering = LRProcessOrdering.PLANNED; - boolean changeTypeOfOrdering = this.ordering.equals(nOrdering); - return newOrderingURL(nOrdering, startedString, - changeTypeOfOrdering ? switchOrdering() - : TypeOfOrdering.ASC); - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return e.getMessage(); - } - } - - public boolean isFinishedDateOrdered() { - return this.ordering.equals(LRProcessOrdering.FINISHED); - } - - public String getFinishedDateOrdering() { - try { - String startedString = bundleService.getResourceBundle("labels", - this.localesProvider.get()).getString( - "administrator.processes.finished"); - LRProcessOrdering nOrdering = LRProcessOrdering.FINISHED; - boolean changeTypeOfOrdering = this.ordering.equals(nOrdering); - return newOrderingURL(nOrdering, startedString, - changeTypeOfOrdering ? switchOrdering() - : TypeOfOrdering.ASC); - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return e.getMessage(); - } - } - - public boolean isUserOrdered() { - return this.ordering.equals(LRProcessOrdering.LOGINNAME); - } - - public String getUserOrdering() { - try { - String pidString = bundleService.getResourceBundle("labels", - this.localesProvider.get()).getString( - "administrator.processes.user"); - LRProcessOrdering nOrdering = LRProcessOrdering.LOGINNAME; - boolean changeTypeOfOrdering = this.ordering.equals(nOrdering); - return newOrderingURL(nOrdering, pidString, - changeTypeOfOrdering ? switchOrdering() - : TypeOfOrdering.ASC); - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return e.getMessage(); - } - } - - public String getPidOrdering() { - try { - String pidString = bundleService.getResourceBundle("labels", - this.localesProvider.get()).getString( - "administrator.processes.pid"); - LRProcessOrdering nOrdering = LRProcessOrdering.ID; - boolean changeTypeOfOrdering = this.ordering.equals(nOrdering); - return newOrderingURL(nOrdering, pidString, - changeTypeOfOrdering ? switchOrdering() - : TypeOfOrdering.ASC); - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return e.getMessage(); - } - } - - public boolean isStateOrdered() { - return this.ordering.equals(LRProcessOrdering.STATE); - } - - public String getStateOrdering() { - try { - String stateString = bundleService.getResourceBundle("labels", - this.localesProvider.get()).getString( - "administrator.processes.state"); - LRProcessOrdering nOrdering = LRProcessOrdering.STATE; - boolean changeTypeOfOrdering = this.ordering.equals(nOrdering); - return newOrderingURL(nOrdering, stateString, - changeTypeOfOrdering ? switchOrdering() - : TypeOfOrdering.ASC); - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return e.getMessage(); - } - } - - public boolean isBatchStateOrdered() { - return this.ordering.equals(LRProcessOrdering.BATCHSTATE); - } - - public String getBatchStateOrdering() { - try { - String stateString = bundleService.getResourceBundle("labels", - this.localesProvider.get()).getString( - "administrator.processes.batch"); - LRProcessOrdering nOrdering = LRProcessOrdering.BATCHSTATE; - boolean changeTypeOfOrdering = this.ordering.equals(nOrdering); - return newOrderingURL(nOrdering, stateString, - changeTypeOfOrdering ? switchOrdering() - : TypeOfOrdering.ASC); - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return e.getMessage(); - } - } - - private String newOrderingURL(LRProcessOrdering nOrdering, String name, - TypeOfOrdering ntypeOfOrdering) { - String href = ""; - return href; - } - - public String getPlannedAfter() { - if (this.filter != null) { - List tripples = this.filter.getTripples(); - for (Tripple tripple : tripples) { - if (tripple.getName().equals("planned") - && tripple.getOp().equals(SQLFilter.Op.GT)) { - return this.filter.getFormattedValue(tripple); - } - } - } - return ""; - } - - public String getPlannedBefore() { - if (this.filter != null) { - List tripples = this.filter.getTripples(); - for (Tripple tripple : tripples) { - if (tripple.getName().equals("planned") - && tripple.getOp().equals(SQLFilter.Op.LT)) { - - return this.filter.getFormattedValue(tripple); - } - } - } - return ""; - } - - public String getStartedAfter() { - if (this.filter != null) { - List tripples = this.filter.getTripples(); - for (Tripple tripple : tripples) { - if (tripple.getName().equals("started") - && tripple.getOp().equals(SQLFilter.Op.GT)) { - return this.filter.getFormattedValue(tripple); - } - } - } - return ""; - } - - public String getStartedBefore() { - if (this.filter != null) { - List tripples = this.filter.getTripples(); - for (Tripple tripple : tripples) { - if (tripple.getName().equals("started") - && tripple.getOp().equals(SQLFilter.Op.LT)) { - return this.filter.getFormattedValue(tripple); - } - } - } - return ""; - } - - public String getFinishedAfter() { - if (this.filter != null) { - List tripples = this.filter.getTripples(); - for (Tripple tripple : tripples) { - if (tripple.getName().equals("finished") - && tripple.getOp().equals(SQLFilter.Op.GT)) { - return this.filter.getFormattedValue(tripple); - } - } - } - return ""; - } - - public String getFinishedBefore() { - if (this.filter != null) { - List tripples = this.filter.getTripples(); - for (Tripple tripple : tripples) { - if (tripple.getName().equals("finished") - && tripple.getOp().equals(SQLFilter.Op.LT)) { - return this.filter.getFormattedValue(tripple); - } - } - } - return ""; - } - - public String getLoginNameLike() { - if (this.filter != null) { - List tripples = this.filter.getTripples(); - for (Tripple tripple : tripples) { - if (tripple.getName().equals("loginname") && tripple.getOp().equals(SQLFilter.Op.LIKE)) { - return this.filter.getFormattedValue(tripple); - } - } - } - return ""; - } - - public String getNameLike() { - if (this.filter != null) { - List tripples = this.filter.getTripples(); - for (Tripple tripple : tripples) { - if (tripple.getName().equals("name") - && tripple.getOp().equals(SQLFilter.Op.LIKE)) { - return this.filter.getFormattedValue(tripple); - } - } - } - return ""; - } - - public List getBatchStatesForFilter() { - List wrap = BatchProcessStateWrapper.wrap( - true, BatchStates.values()); - if (this.filter != null) { - Tripple tripple = this.filter.findTripple("batch_status"); - if (tripple != null) { - Integer intg = (Integer) tripple.getVal(); - if (intg.intValue() >= 0) { - for (BatchProcessStateWrapper wrapper : wrap) { - if (wrapper.getVal() == intg.intValue()) { - wrapper.setSelected(true); - } - } - } - } - } - return wrap; - } - - public List getStatesForFilter() { - List wrap = ProcessStateWrapper.wrap(true, - States.values()); - if (this.filter != null) { - Tripple tripple = this.filter.findTripple("status"); - if (tripple != null) { - Integer intg = (Integer) tripple.getVal(); - if (intg.intValue() >= 0) { - for (ProcessStateWrapper wrapper : wrap) { - if (wrapper.getVal() == intg.intValue()) { - wrapper.setSelected(true); - } - } - } - } - } - return wrap; - } - - private String orderingImg(LRProcessOrdering nOrdering) { - if (nOrdering.equals(this.ordering)) { - if (typeOfOrdering.equals(TypeOfOrdering.DESC)) { - return ""; - } else { - return ""; - } - } else - return ""; - } - - public LRProcessManager getProcessManager() { - return processManager; - } - - public void setProcessManager(LRProcessManager processManager) { - this.processManager = processManager; - } - - public DefinitionManager getDefinitionManager() { - return definitionManager; - } - - public void setDefinitionManager(DefinitionManager definitionManager) { - this.definitionManager = definitionManager; - } - - public ResourceBundleService getBundleService() { - return bundleService; - } - - public void setBundleService(ResourceBundleService bundleService) { - this.bundleService = bundleService; - } - - public Provider getLocalesProvider() { - return localesProvider; - } - - public void setLocalesProvider(Provider localesProvider) { - this.localesProvider = localesProvider; - } - - public Provider getRequestProvider() { - return requestProvider; - } - - public void setRequestProvider(Provider requestProvider) { - this.requestProvider = requestProvider; - } - - public boolean isCurrentFirstPage() { - return getPage() == getNumberOfPages()-1; - } - - public boolean isCurrentLastPage() { - return getPage() == 0; - } - -} diff --git a/search/src/java/cz/incad/Kramerius/views/SearchFormViewObject.java b/search/src/java/cz/incad/Kramerius/views/SearchFormViewObject.java deleted file mode 100644 index 87d005ade9..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/SearchFormViewObject.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views; - -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; - -import javax.servlet.http.HttpServletRequest; - -import com.google.inject.Inject; -import com.google.inject.Provider; - -public class SearchFormViewObject { - - @Inject - Provider requestProvider; - - public String getRequestedAddress() throws UnsupportedEncodingException { - String retValue = this.requestProvider.get().getRequestURL().toString(); - String queryString = this.requestProvider.get().getQueryString(); - - if (queryString != null) { - retValue = retValue + "?" +queryString; - } - - return URLEncoder.encode(retValue, "UTF-8"); - } - - -} diff --git a/search/src/java/cz/incad/Kramerius/views/StaticExportViewObject.java b/search/src/java/cz/incad/Kramerius/views/StaticExportViewObject.java deleted file mode 100644 index dd2c9c143b..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/StaticExportViewObject.java +++ /dev/null @@ -1,21 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views; - -public class StaticExportViewObject extends AbstractPrintViewObject { - -} diff --git a/search/src/java/cz/incad/Kramerius/views/WellcomeViewObject.java b/search/src/java/cz/incad/Kramerius/views/WellcomeViewObject.java deleted file mode 100644 index a82975e7ec..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/WellcomeViewObject.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.util.Locale; - -import org.antlr.stringtemplate.StringTemplate; -import org.antlr.stringtemplate.StringTemplateGroup; -import org.antlr.stringtemplate.language.DefaultTemplateLexer; - -import com.google.inject.Inject; -import com.google.inject.Provider; - -import cz.incad.kramerius.ObjectPidsPath; -import cz.incad.kramerius.security.RightsResolver; -import cz.incad.kramerius.security.SecuredActions; -import cz.incad.kramerius.security.SpecialObjects; -import cz.incad.kramerius.service.TextsService; - -public class WellcomeViewObject { - - private static final String DEFAULT_INTRO_CONSTANT = "default_intro"; - - private static final String INTRO_CONSTANT = "intro"; - - static java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(WellcomeViewObject.class.getName()); - - @Inject - TextsService textService; - - @Inject - Provider provider; - - @Inject - RightsResolver rightsResolver; - - public String getIntro() throws IOException { - boolean operationPermited = rightsResolver.isActionAllowed(SecuredActions.EDIT_INFO_TEXT.getFormalName(), SpecialObjects.REPOSITORY.getPid(), null, ObjectPidsPath.REPOSITORY_PATH).flag(); - return operationPermited ? getTextIntro() : getTextIntro(); - } - - public String getEditIntro() throws IOException { - StringTemplate template = stGroup().getInstanceOf("editor"); - template.setAttribute("text", getTextIntro()); - template.setAttribute("lang", provider.get().getLanguage()); - return template.toString(); - } - - - public String getTextIntro() throws IOException { - if (textService.isAvailable(INTRO_CONSTANT, provider.get())) { - return textService.getText(INTRO_CONSTANT, provider.get()); - } else { - return textService.getText(DEFAULT_INTRO_CONSTANT,provider.get()); - } - } - - - private static StringTemplateGroup stGroup() { - InputStream is = WellcomeViewObject.class.getResourceAsStream("wellcomehtml.stg"); - StringTemplateGroup grp = new StringTemplateGroup(new InputStreamReader(is), DefaultTemplateLexer.class); - return grp; - } - -} diff --git a/search/src/java/cz/incad/Kramerius/views/inc/details/tabs/AltoSupportViewObject.java b/search/src/java/cz/incad/Kramerius/views/inc/details/tabs/AltoSupportViewObject.java deleted file mode 100644 index b25c9b82d9..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/inc/details/tabs/AltoSupportViewObject.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright (C) 2012 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -/** - * - */ -package cz.incad.Kramerius.views.inc.details.tabs; - -import java.io.IOException; -import java.io.InputStream; -import java.util.Map; -import java.util.logging.Level; - -import javax.servlet.http.HttpServletRequest; -import javax.xml.parsers.ParserConfigurationException; - -import net.sf.json.JSONObject; - -import org.w3c.dom.Document; -import org.xml.sax.SAXException; - -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.google.inject.name.Named; - -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.Initializable; -import cz.incad.kramerius.utils.ALTOUtils; -import cz.incad.kramerius.utils.ALTOUtils.AltoDisected; -import cz.incad.kramerius.utils.XMLUtils; - -/** - * @author pavels - * - */ -public class AltoSupportViewObject implements Initializable { - - static java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(AltoSupportViewObject.class.getName()); - - private static final String PID = "pid"; - - @Inject - @Named("securedFedoraAccess") - FedoraAccess fedoraAccess; - - @Inject - Provider requestProvider; - - - private String imagePid = null; - - @Override - public void init() { - try { - HttpServletRequest request = this.requestProvider.get(); - String pid = request.getParameter(PID); - this.imagePid = fedoraAccess.findFirstViewablePid(pid); - LOGGER.fine("processing pid '"+this.imagePid+"'"); - } catch (IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } - } - - - - public String getSearchQuery() { - HttpServletRequest request = this.requestProvider.get(); - if (request.getParameterMap().containsKey("q")) { - String par = request.getParameter("q"); - par = par.replace("'", "\\'"); - return "'"+par+"'"; - } else return "null"; - } - - public boolean getAltoStreamAvailabilityFlag() throws IOException { - boolean streamAvailable = fedoraAccess.isStreamAvailable(this.imagePid, "ALTO"); - return streamAvailable; - } - - public boolean getQueryAvailabilityFlag() { - HttpServletRequest request = this.requestProvider.get(); - return (request.getParameterMap().containsKey("q")); - } - - - private Document getAltoDocument() throws IOException, ParserConfigurationException, SAXException { - InputStream is = this.fedoraAccess.getDataStream(this.imagePid, "ALTO"); - return XMLUtils.parseDocument(is); - } - - public String getAltoJSONInitialization() throws IOException, ParserConfigurationException, SAXException { - HttpServletRequest request = this.requestProvider.get(); - if (request.getParameterMap().containsKey("q")) { - String par = request.getParameter("q"); - if (getAltoStreamAvailabilityFlag()) { - Document parsed = getAltoDocument(); - AltoDisected disected = ALTOUtils.disectAlto(par, parsed); - JSONObject jsonObj = new JSONObject(); - jsonObj.put("alto", disected.toJSON()); - return jsonObj.toString(); - } - } - return ""; - } - -} diff --git a/search/src/java/cz/incad/Kramerius/views/inc/details/tabs/AudioViewObject.java b/search/src/java/cz/incad/Kramerius/views/inc/details/tabs/AudioViewObject.java deleted file mode 100644 index 035fbf1aa7..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/inc/details/tabs/AudioViewObject.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright (C) 2010 jenyk.holman - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views.inc.details.tabs; - -import java.io.IOException; -import java.util.Locale; -import javax.servlet.http.HttpServletRequest; -import com.google.inject.Inject; -import com.google.inject.Provider; -import cz.incad.kramerius.service.ResourceBundleService; -import cz.incad.kramerius.service.TextsService; - - -public class AudioViewObject { - - private static final String RIGHT_MSG = "rightMsg"; - - private static final String PLAYER_NOT_SUPPORTED_MSG = "notSupportedMsg"; - - @Inject - TextsService textsService; - - @Inject - ResourceBundleService resourceBundleService; - - @Inject - Provider localeProvider; - - @Inject - Provider requestProvider; - - public String getNotAllowedMessageText() throws IOException { - Locale locale = this.localeProvider.get(); - if (textsService.isAvailable(RIGHT_MSG, locale)) { - return textsService.getText(RIGHT_MSG, locale); - } else return this.resourceBundleService.getResourceBundle("labels", locale).getString(RIGHT_MSG); - } - - public String getNotSupportedMessageText() throws IOException { - Locale locale = this.localeProvider.get(); - return this.resourceBundleService.getResourceBundle("labels", locale).getString(PLAYER_NOT_SUPPORTED_MSG); - } - -} diff --git a/search/src/java/cz/incad/Kramerius/views/inc/details/tabs/ImageViewObject.java b/search/src/java/cz/incad/Kramerius/views/inc/details/tabs/ImageViewObject.java deleted file mode 100644 index 65a3ca282d..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/inc/details/tabs/ImageViewObject.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views.inc.details.tabs; - -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.google.inject.name.Named; -import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.security.RightsResolver; -import cz.incad.kramerius.security.SecuredActions; -import cz.incad.kramerius.service.ResourceBundleService; -import cz.incad.kramerius.service.TextsService; -import cz.incad.kramerius.utils.conf.KConfiguration; -import cz.incad.kramerius.utils.pid.LexerException; -import cz.incad.kramerius.utils.pid.PIDParser; - -import javax.servlet.http.HttpServletRequest; -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; -import java.util.logging.Level; - -/** - * View object for images - * @author pavels - */ -public class ImageViewObject { - - static java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(ImageViewObject.class.getName()); - - private static final String PID = "pid"; - - private static final String RIGHT_MSG = "rightMsg"; - private static final String RIGHT_MSG_ALTERNATIVE = "rightMsgAlternative"; - - @Inject - TextsService textsService; - - @Inject - ResourceBundleService resourceBundleService; - - @Inject - Provider localeProvider; - - @Inject - Provider requestProvider; - - - @Inject - KConfiguration configuration; - - @Inject - RightsResolver rightsResolver; - - @Inject - @Named("new-index") - SolrAccess solrAccess; - - public String getNotAllowedMessageText() throws IOException { - Locale locale = this.localeProvider.get(); - String pid = requestProvider.get().getParameter(PID); - if(pid != null - && rightsResolver.isActionAllowed(SecuredActions.SHOW_ALTERNATIVE_INFO_TEXT.getFormalName(), pid, null, solrAccess.getPidPaths(pid)[0]).flag() - && textsService.isAvailable(RIGHT_MSG_ALTERNATIVE,locale)){ - return replaceUuidInMessage(textsService.getText(RIGHT_MSG_ALTERNATIVE, locale), pid); - } else if (textsService.isAvailable(RIGHT_MSG, locale)) { - return replaceUuidInMessage(textsService.getText(RIGHT_MSG, locale), pid); - } else return this.resourceBundleService.getResourceBundle("labels", locale).getString(RIGHT_MSG); - } - - - public int getPageInt() { - return isPagePid() ? Integer.parseInt(getPage())-1 : 0; - } - - public String getPage() { - try { - String pidParam = this.requestProvider.get().getParameter(PID); - if (pidParam != null) { - PIDParser pidParser = new PIDParser(pidParam); - pidParser.objectPid(); - if (pidParser.isPagePid()) { - return pidParser.getPage(); - } - } - } catch (LexerException e) { - LOGGER.log(Level.SEVERE, e.getMessage(),e); - } - return ""; - } - - - public boolean isPagePid() { - try { - String pidParam = this.requestProvider.get().getParameter(PID); - if (pidParam != null) { - PIDParser pidParser = new PIDParser(pidParam); - pidParser.objectPid(); - return pidParser.isPagePid(); - } else return false; - } catch (LexerException e) { - LOGGER.log(Level.SEVERE, e.getMessage(),e); - return false; - } - } - - - public ZoomViewer getZoomingViewer() { - // There are only two options : zoomify and deepzoom - String zoomViewer = this.configuration.getConfiguration().getString("zoom.viewer", "zoomify"); - return ZoomViewer.valueOf(zoomViewer); - } - - public List getScripturls() { - return getZoomingViewer().getScripturls(); - } - - public String getDivContainer() { - return getZoomingViewer().getDivContainer(); - } - - private String replaceUuidInMessage(String message, String pid){ - if(message.contains("${uuid}")) { - pid = (pid == null ? "" : pid); - message = message.replace("${uuid}",pid); - } - return message; - } - - static enum ZoomViewer { - - zoomify(new String[] {"js/zoom/OpenLayers.js","js/zoom/zoomify.js"},"ol-container"), - deepzoom( new String[] {"js/zoom/deepzoom.js"}, "container"); - - private String[] scripturls; - private String divcontainer; - - private ZoomViewer(String[] scripturls, String divcontainer) { - this.scripturls = scripturls; - this.divcontainer = divcontainer; - } - - /** - * @return the scripturls - */ - public List getScripturls() { - return Arrays.asList(scripturls); - } - - /** - * @return the divcontainer - */ - public String getDivContainer() { - return divcontainer; - } - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/inc/details/tabs/LoadCustomViewObject.java b/search/src/java/cz/incad/Kramerius/views/inc/details/tabs/LoadCustomViewObject.java deleted file mode 100644 index 9251948019..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/inc/details/tabs/LoadCustomViewObject.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views.inc.details.tabs; - -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.google.inject.name.Named; - -import cz.incad.Kramerius.I18NServlet; -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.Initializable; -import cz.incad.kramerius.service.ResourceBundleService; -import cz.incad.kramerius.service.XSLService; -import cz.incad.kramerius.utils.IOUtils; -import cz.incad.kramerius.utils.UnicodeUtil; -import cz.incad.kramerius.utils.XMLUtils; - -import org.apache.commons.lang3.StringEscapeUtils; -import org.xml.sax.SAXException; - -import javax.servlet.http.HttpServletRequest; -import javax.xml.parsers.ParserConfigurationException; - -import java.io.ByteArrayInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.Charset; -import java.util.*; -import java.util.logging.Level; - -public class LoadCustomViewObject implements Initializable { - - static java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(LoadCustomViewObject.class.getName()); - - @Inject - Provider requestProvider; - - - @Inject - Provider localesProvider; - - @Inject - @Named("securedFedoraAccess") - FedoraAccess fedoraAccess; - - @Inject - XSLService xslService; - - @Inject - ResourceBundleService resourceBundleService; - - - @Override - public void init() { - } - - - public String getI18NServlet() { - String i18nServlet = I18NServlet.i18nServlet(requestProvider.get()) + "?action=bundle&lang=" + localesProvider.get().getLanguage() + "&country=" + localesProvider.get().getCountry() + "&name=labels"; - return i18nServlet; - } - - private String escapeXML(String s) { - return s.replaceAll("<", "<").replaceAll(">", ">").replaceAll("'", "'").replaceAll("\"", """).replaceAll("&", "&"); - } - - - // rewritten from jsp - no changes - public String getContent() throws IOException, ParserConfigurationException, SAXException { - StringBuilder stringBuilder = new StringBuilder(); - - String tab = this.requestProvider.get().getParameter("tab"); - String ds = tab; - String xsl = tab; - if (tab.indexOf('.') >= 0) { - ds = tab.split("\\.")[0]; - xsl = tab.split("\\.")[1] + ".xsl"; - } - - String pid_path = this.requestProvider.get().getParameter("pid_path"); - List pids = Arrays.asList(pid_path.split("/")); - if (ds.startsWith("-")) { - Collections.reverse(pids); - ds = ds.substring(1); - } - for (String pid : pids) { - if (fedoraAccess.isStreamAvailable(pid, ds)) { - - String mime = fedoraAccess.getMimeTypeForStream(pid, ds); - if (mime.equals("text/plain")) { - try { - InputStream is = fedoraAccess.getDataStream(pid, ds); - byte[] bytes = org.apache.commons.io.IOUtils.toByteArray(is); - String enc = UnicodeUtil.getEncoding(bytes); - ByteArrayInputStream is2 = new ByteArrayInputStream(bytes); - stringBuilder.append(""); - } catch (cz.incad.kramerius.security.SecurityException e) { - LOGGER.log(Level.INFO, e.getMessage()); - } - } else if (mime.equals("text/xml") || mime.equals("application/rdf+xml")) { - try { - if (xslService.isAvailable(xsl)) { - org.w3c.dom.Document xml = XMLUtils.parseDocument(fedoraAccess.getDataStream(pid, ds), true); - String text = xslService.transform(xml, xsl, this.localesProvider.get()); - stringBuilder.append(text); - } else { - String xmltext = org.apache.commons.io.IOUtils.toString(fedoraAccess.getDataStream(pid, ds), Charset.forName("UTF-8")); - stringBuilder.append(StringEscapeUtils.escapeHtml4(xmltext)); - } - } catch (cz.incad.kramerius.security.SecurityException e) { - LOGGER.log(Level.INFO, e.getMessage()); - } catch (Exception e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } - } else if (mime.equals("text/html")) { - try { - String xmltext = org.apache.commons.io.IOUtils.toString(fedoraAccess.getDataStream(pid, ds), Charset.forName("UTF-8")); - stringBuilder.append(xmltext); - } catch (cz.incad.kramerius.security.SecurityException e) { - LOGGER.log(Level.INFO, e.getMessage()); - } catch (Exception e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } - } - } - } - return stringBuilder.toString(); - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/localprint/AbstractPrepareViewObject.java b/search/src/java/cz/incad/Kramerius/views/localprint/AbstractPrepareViewObject.java deleted file mode 100644 index bfe4ea357a..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/localprint/AbstractPrepareViewObject.java +++ /dev/null @@ -1,53 +0,0 @@ -package cz.incad.Kramerius.views.localprint; - -import java.awt.Dimension; -import java.util.ArrayList; -import java.util.List; - -public class AbstractPrepareViewObject { - - protected List imgelements = new ArrayList(); - protected List styles = new ArrayList(); - - public List getStyles() { - return this.styles; - } - - public List getImgelements() { - return imgelements; - } - - protected String width(String ident) { - return "#"+ident+"{ width:100%;}"; - } - - protected String height(String ident) { - return "#"+ident+"{ height:100%;}"; - } - - protected String createIdent(int i, int bits) { - StringBuilder builder = new StringBuilder(); - for (int j = 0; j < bits; j++) { - int v = (i >> j) & 1; - builder.append(v==0 ? 'a':'z'); - } - return builder.toString(); - } - - protected int numberOfBits(int length) { - double bits = Math.floor( Math.pow(length,0.5)); - if (Math.pow(bits, 2) < length) { - bits = bits +1; - } - return (int) bits; - } - - protected void createStyle(double ratio, String ident, Dimension readDim) { - double calculated = (double)readDim.height/(double)readDim.width; - if (calculated< ratio) { - this.styles.add(width(ident)); - } else { - this.styles.add(height(ident)); - } - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/localprint/DisectionPrepareViewObject.java b/search/src/java/cz/incad/Kramerius/views/localprint/DisectionPrepareViewObject.java deleted file mode 100644 index f1e985cb86..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/localprint/DisectionPrepareViewObject.java +++ /dev/null @@ -1,73 +0,0 @@ -package cz.incad.Kramerius.views.localprint; - -import java.awt.Dimension; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.List; - -import javax.servlet.http.HttpServletRequest; -import javax.xml.xpath.XPathExpressionException; - -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.google.inject.name.Named; - -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.Initializable; -import cz.incad.kramerius.utils.conf.KConfiguration; -import cz.incad.kramerius.utils.imgs.KrameriusImageSupport; - -public class DisectionPrepareViewObject extends AbstractPrepareViewObject implements Initializable{ - - @Inject - Provider servletRequestProvider; - - @Inject - @Named("securedFedoraAccess") - FedoraAccess fedoraAccess; - - private List pids = new ArrayList(); - - - @Override - public void init() { - try { - double ratio = KConfiguration.getInstance().getConfiguration().getDouble("search.print.pageratio",1.414); - HttpServletRequest request = this.servletRequestProvider.get(); - - String pidsString = request.getParameter("pid"); - pidsString = this.fedoraAccess.findFirstViewablePid(pidsString); - String transcode = request.getParameter("transcode"); - String width = request.getParameter("width"); - String height = request.getParameter("height"); - String xpos = request.getParameter("xpos"); - String ypos = request.getParameter("ypos"); - - int bits = numberOfBits(1); - - - String ident = createIdent(0,bits); - this.pids.add(URLDecoder.decode(pidsString, "UTF-8")); - - String url ="../imgcut?pid="+URLEncoder.encode(pidsString,"UTF-8")+"&xpos="+xpos+"&ypos="+ypos+"&width="+width+"&height="+height; - String imageElement = ""; - this.imgelements.add(imageElement); - - double dwidth = Double.parseDouble(width); - double dheight = Double.parseDouble(height); - int iwidth = (int)(dwidth * 1000); - int iheight = (int)(dheight *1000); - - Dimension readDim = new Dimension(iwidth, iheight); - createStyle(ratio, ident, readDim); - - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new RuntimeException(e); - } - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/localprint/PicturePrepareViewObject.java b/search/src/java/cz/incad/Kramerius/views/localprint/PicturePrepareViewObject.java deleted file mode 100644 index db9f395b1b..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/localprint/PicturePrepareViewObject.java +++ /dev/null @@ -1,174 +0,0 @@ -package cz.incad.Kramerius.views.localprint; - -import java.awt.Dimension; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.net.URLDecoder; -import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.List; -import java.util.logging.Logger; - -import javax.servlet.http.HttpServletRequest; -import javax.xml.xpath.XPathExpressionException; - -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.google.inject.name.Named; - -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.Initializable; -import cz.incad.kramerius.ObjectPidsPath; -import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.rest.api.k5.client.SolrMemoization; -import cz.incad.kramerius.rest.api.k5.client.item.utils.ItemResourceUtils; -import cz.incad.kramerius.security.RightsResolver; -import cz.incad.kramerius.security.SecuredActions; -import cz.incad.kramerius.security.User; -import cz.incad.kramerius.utils.StringUtils; -import cz.incad.kramerius.utils.conf.KConfiguration; -import cz.incad.kramerius.utils.imgs.KrameriusImageSupport; - -public class PicturePrepareViewObject extends AbstractPrepareViewObject implements Initializable{ - - public static final Logger LOGGER = Logger.getLogger(PicturePrepareViewObject.class.getName()); - - @Inject - Provider servletRequestProvider; - - @Inject - @Named("securedFedoraAccess") - FedoraAccess fedoraAccess; - - @Inject - Provider userProvider; - - @Inject - RightsResolver rightsResolver; - - @Inject - @Named("new-index") - SolrAccess solrAccess; - - @Inject - SolrMemoization solrMemoization; - - private List pids = new ArrayList(); - - - @Override - public void init() { - try { - double ratio = KConfiguration.getInstance().getConfiguration().getDouble("search.print.pageratio",1.414); - HttpServletRequest request = this.servletRequestProvider.get(); - - String pidsString = request.getParameter("pids"); - String startPid = request.getParameter("startPid"); - String stopPid = request.getParameter("stopPid"); - String parentPid = request.getParameter("parentPid"); - - String[] pids = new String[0]; - if (StringUtils.isAnyString(pidsString)) { - pids = pidsAsList(pidsString); - } else if (StringUtils.isAnyString(startPid)){ - pids = pidsAsSiblings(startPid, stopPid); - } else if (StringUtils.isAnyString(parentPid)){ - pids = pidsAsChildren(parentPid); - } - - String transcode = request.getParameter("transcode"); - int bits = numberOfBits(pids.length); - for (int i = 0; i < pids.length; i++) { - String p = pids[i]; - boolean canBeRead = canBeRead(p); - if (canBeRead) { - p = this.fedoraAccess.findFirstViewablePid(p); - String ident = createIdent(i,bits); - this.pids.add(URLDecoder.decode(p, "UTF-8")); - - String url ="../img?pid="+URLEncoder.encode(p,"UTF-8")+"&stream=IMG_FULL&action="+(Boolean.parseBoolean(transcode) ? "TRANSCODE":"GETRAW"); - String imageElement = ""; - this.imgelements.add(imageElement); - - Dimension readDim = KrameriusImageSupport.readDimension(p, "IMG_FULL", fedoraAccess, 0); - LOGGER.fine("Dimension is :"+readDim); - createStyle(ratio, ident, readDim); - } - } - } catch (UnsupportedEncodingException e) { - throw new RuntimeException(e); - } catch (XPathExpressionException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - - - private String[] pidsAsChildren(String parentPid) - throws IOException { - String[] pids = new String[0]; - List solrChildrenPids = ItemResourceUtils.solrChildrenPids(parentPid, new ArrayList(), this.solrAccess, this.solrMemoization); - return solrChildrenPids.toArray(new String[solrChildrenPids.size()]); - } - - - private String[] pidsAsSiblings( String startPid, String stopPid) - throws IOException { - String[] pids = new String[0]; - ObjectPidsPath[] paths = this.solrAccess.getPidPaths(startPid); - ObjectPidsPath pths = selectOnePath(startPid, paths); - String[] pidsPths = pths.getPathFromRootToLeaf(); - if (pidsPths.length > 1) { - String parent = pidsPths[pidsPths.length -2]; - List solrChildrenPids = ItemResourceUtils.solrChildrenPids(parent, new ArrayList(), this.solrAccess, this.solrMemoization); - List rest = new ArrayList(); - boolean append = false; - for (String pid : solrChildrenPids) { - if ((!append) && pid.equals(startPid)) { - append = true; - } - if (append) { - rest.add(pid); - } - if ((append) && pid.equals(stopPid)) { - append = false; - break; - } - } - pids = rest.toArray(new String[rest.size()]); - } - return pids; - } - - - private String[] pidsAsList(String pidsString) { - String[] pids; - pids = pidsString.split(","); - return pids; - } - - - private boolean canBeRead(String pid) throws IOException { - ObjectPidsPath[] paths = solrAccess.getPidPaths(pid); - for (ObjectPidsPath pth : paths) { - if (this.rightsResolver.isActionAllowed(userProvider.get(), SecuredActions.READ.getFormalName(), pid, null, pth.injectRepository()).flag()) { - return true; - } - } - return false; - } - - public static ObjectPidsPath selectOnePath(String requestedPid, - ObjectPidsPath[] paths) { - ObjectPidsPath path; - if (paths.length > 0) { - path = paths[0]; - } else { - path = new ObjectPidsPath(requestedPid); - } - return path; - } - -} - diff --git a/search/src/java/cz/incad/Kramerius/views/rights/AbstractRightsView.java b/search/src/java/cz/incad/Kramerius/views/rights/AbstractRightsView.java deleted file mode 100644 index 355daf2a4f..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/rights/AbstractRightsView.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views.rights; - -import java.util.ArrayList; - -import javax.servlet.http.HttpServletRequest; - -import antlr.RecognitionException; -import antlr.TokenStreamException; - -import com.google.inject.Inject; -import com.google.inject.Provider; - -import cz.incad.Kramerius.views.AbstractViewObject; -import cz.incad.kramerius.security.Role; -import cz.incad.kramerius.security.User; - -public class AbstractRightsView extends AbstractViewObject { - - public static final String RIGHT_ID="ids"; - - - public String getRightIdParam() throws RecognitionException, TokenStreamException { - HttpServletRequest httpServletRequest = this.requestProvider.get(); - String parameter = httpServletRequest.getParameter(RIGHT_ID); - return parameter; - } - - public String getSecuredAction() { - String securedActionString = this.requestProvider.get().getParameter("securedaction"); - return securedActionString; - } - - protected static boolean hasSuperAdminRole(User user) { - Role[] grps = user.getGroups(); - for (Role grp : grps) { - if (grp.getPersonalAdminId() == 0) { - return true; - } - } - return false; - } - -} diff --git a/search/src/java/cz/incad/Kramerius/views/rights/CriteriumsManageView.java b/search/src/java/cz/incad/Kramerius/views/rights/CriteriumsManageView.java deleted file mode 100644 index 733737d60d..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/rights/CriteriumsManageView.java +++ /dev/null @@ -1,141 +0,0 @@ -/* - * Copyright (C) 2012 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views.rights; - -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import javax.servlet.http.HttpServletRequest; - -import org.antlr.stringtemplate.StringTemplate; - -import com.google.inject.Inject; -import com.google.inject.Provider; - -import cz.incad.Kramerius.security.strenderers.CriteriumParamsWrapper; -import cz.incad.kramerius.security.RightCriteriumParams; -import cz.incad.kramerius.security.RightsManager; -import cz.incad.kramerius.security.User; -import cz.incad.kramerius.security.UserManager; -import cz.incad.kramerius.utils.ApplicationURL; - - -public class CriteriumsManageView extends AbstractRightsView { - - - @Inject - UserManager userManager; - - @Inject - Provider userProvider; - - @Inject - RightsManager rightsManager; - - @Inject - Provider requestProvider; - - - public _RightCriteriumParams[] getCriteriumParams() { - RightCriteriumParams[] allParams = this.rightsManager.findAllParams(); - List<_RightCriteriumParams> list = new ArrayList(); - for (RightCriteriumParams param : allParams) { - List> objects = this.rightsManager.findObjectUsingParams(param.getId()); - - - list.add(new _RightCriteriumParams(objects,param)); - } - return (_RightCriteriumParams[]) list.toArray(new _RightCriteriumParams[list.size()]); - } - - - public String getApplicationURL() { - String appURL = ApplicationURL.applicationURL(this.requestProvider.get()); - return appURL; - } - - - /** criterium param item */ - public static class _RightCriteriumParams implements RightCriteriumParams { - - private RightCriteriumParams params = null; - private List> objects = null; - - public _RightCriteriumParams(List> objects, RightCriteriumParams params) { - super(); - this.params = params; - this.objects = objects; - } - - public int getId() { - return params.getId(); - } - - public Object[] getObjects() { - return params.getObjects(); - } - - public void setObjects(Object[] objs) { - params.setObjects(objs); - } - - public String getObjectsString() { - StringTemplate tmpl = new StringTemplate("$objects;separator=\";\"$"); - tmpl.setAttribute("objects", this.getObjects()); - return tmpl.toString(); - } - - - public String getLongDescription() { - return params.getLongDescription(); - } - - public void setLongDescription(String longDesc) { - params.setLongDescription(longDesc); - } - - public String getShortDescription() { - return params.getShortDescription(); - } - - public void setShortDescription(String desc) { - params.setShortDescription(desc); - } - - public void setId(int id) { - params.setId(id); - } - - - public List getUsedPids() { - List pids = new ArrayList(); - for (Map str : this.objects) { - pids.add(str.get("pid")); - } - return pids; - } - - - public String getUsingObjectsAsJSArray() { - StringTemplate template = new StringTemplate("[ $objs:{obj| \\{'pid':'$obj.pid$', 'action':'$obj.action$'\\} };separator=\",\"$ ]"); - template.setAttribute("objs", this.objects); - return template.toString(); - } - - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/rights/DisplayObjectsView.java b/search/src/java/cz/incad/Kramerius/views/rights/DisplayObjectsView.java deleted file mode 100644 index 72835681da..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/rights/DisplayObjectsView.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views.rights; - - -import static cz.incad.kramerius.security.SecuredActions.ADMINISTRATE; -import static cz.incad.kramerius.security.SecuredActions.EXPORT_K4_REPLICATIONS; -import static cz.incad.kramerius.security.SecuredActions.READ; - -import java.io.IOException; -import java.io.StringReader; -import java.util.ArrayList; -import java.util.List; -import java.util.Locale; -import java.util.logging.Level; - -import javax.inject.Named; -import javax.servlet.http.HttpServletRequest; - -import org.w3c.dom.Document; - -import antlr.RecognitionException; -import antlr.TokenStreamException; - -import com.google.inject.Inject; -import com.google.inject.Provider; - -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.ObjectPidsPath; -import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.security.RightsResolver; -import cz.incad.kramerius.security.RightCriteriumWrapperFactory; -import cz.incad.kramerius.security.RightsManager; -import cz.incad.kramerius.security.SecuredActions; -import cz.incad.kramerius.security.SpecialObjects; -import cz.incad.kramerius.security.User; -import cz.incad.kramerius.service.ResourceBundleService; -import cz.incad.kramerius.utils.DCUtils; -import cz.incad.kramerius.utils.params.ParamsLexer; -import cz.incad.kramerius.utils.params.ParamsParser; -import cz.incad.kramerius.utils.pid.LexerException; -import cz.incad.kramerius.utils.pid.PIDParser; - -public class DisplayObjectsView extends AbstractRightsView { - - public static final String ACTIONS = "actions"; - - static java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(DisplayObjectsView.class.getName()); - - @Inject - @Named("securedFedoraAccess") - FedoraAccess fedoraAccess; - - @Inject - RightsManager rightsManager; - - @Inject - RightsResolver rightsResolver; - - @Inject - Provider userProvider; - - @Inject - @Named("new-index") - SolrAccess solrAccess; - - @Inject - RightCriteriumWrapperFactory factory; - - @Inject - ResourceBundleService resourceBundleService; - - @Inject - Provider localesProvider; - - - - public DisplayObjectsView() { - super(); - } - - - - public String getIdent() { - String idParam = super.requestProvider.get().getParameter("id"); - return idParam; - } - - - public ListgetAffectedObjects() throws LexerException { - List objects = new ArrayList(); - try { - List paramsList = getPidsParams(); - for (Object pid : paramsList) { - boolean hasRight = false; - ObjectPidsPath[] paths = solrAccess.getPidPaths(pid.toString()); - for (ObjectPidsPath path : paths) { - if (this.rightsResolver.isActionAllowed(this.userProvider.get(),SecuredActions.ADMINISTRATE.getFormalName(),pid.toString(), null, path).flag()) { - hasRight = true; - break; - } - } - - - if (pid.equals(SpecialObjects.REPOSITORY.getPid())) { - - String kmodelName = SpecialObjects.REPOSITORY.name(); - AffectedObject affectedObject = new AffectedObject(pid.toString(), SpecialObjects.REPOSITORY.name(),kmodelName, hasRight); - objects.add(affectedObject); - - } else { - PIDParser pidParser = new PIDParser(pid.toString()); - pidParser.objectPid(); - String displayingPid = null; - if (pidParser.isDatastreamPid()) { - displayingPid = pidParser.getParentObjectPid(); - } else { - displayingPid = pid.toString(); - } - - Document dc = this.fedoraAccess.getDC(displayingPid); - - Locale locale = this.localesProvider.get(); - String kmodelName = this.fedoraAccess.getKrameriusModelName(displayingPid); - String translatedKModelName = resourceBundleService.getResourceBundle("labels", locale).getString("document.type."+kmodelName); - - AffectedObject affectedObject = new AffectedObject(pid.toString(), DCUtils.titleFromDC(dc),translatedKModelName, hasRight); - objects.add(affectedObject); - - } - } - - } catch (RecognitionException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } catch (TokenStreamException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } catch (IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } - return objects; - } - - private List getActionsParam() throws RecognitionException, TokenStreamException { - HttpServletRequest httpServletRequest = this.requestProvider.get(); - String parameter = httpServletRequest.getParameter(ACTIONS); - - ParamsParser params = new ParamsParser(new ParamsLexer(new StringReader(parameter))); - List paramsList = params.params(); - return paramsList; - } - - public SecuredActions[] getActions() throws RecognitionException, TokenStreamException { - String parameter = this.requestProvider.get().getParameter(ACTIONS); - if (parameter != null) { - - List actionsParam = getActionsParam(); - List secList = new ArrayList(); - for (Object act : actionsParam) { - SecuredActions action = SecuredActions.findByFormalName(act.toString()); - if (action == null) { - secList.add(SecuredActions.valueOf(act.toString())); - } else { - secList.add(action); - } - } - return (SecuredActions[]) secList.toArray(new SecuredActions[secList.size()]); - - } else { - return new SecuredActions[] {READ,ADMINISTRATE,EXPORT_K4_REPLICATIONS, SecuredActions.SHOW_CLIENT_PRINT_MENU, SecuredActions.SHOW_CLIENT_PDF_MENU, SecuredActions.PDF_RESOURCE}; - } - } - - public class AffectedObject { - - private String pid; - private boolean accessed; - private String title; - private String modelName; - - public AffectedObject(String pid, String title, String modelName, boolean accessed) { - super(); - this.pid = pid; - this.title = title; - this.accessed = accessed; - this.modelName = modelName; - } - - public String getTitle() { - return this.title; - } - - public String getModelName() { - return modelName; - } - - - public String getComment() { - return this.accessed ? "" : "Nemate pravo zit !"; - } - - public String getPid() { - return this.pid; - } - - public boolean isAccessed() { - return this.accessed; - } - - - public String getCheckedAttribute() { - return this.accessed ? "checked='true'" :""; - } - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/rights/DisplayRightView.java b/search/src/java/cz/incad/Kramerius/views/rights/DisplayRightView.java deleted file mode 100644 index c6d6b4b105..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/rights/DisplayRightView.java +++ /dev/null @@ -1,227 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views.rights; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.stream.Collectors; - -import antlr.RecognitionException; -import antlr.TokenStreamException; - -import com.google.inject.Inject; -import com.google.inject.Provider; - -import cz.incad.kramerius.security.*; -import cz.incad.kramerius.security.licenses.License; -import cz.incad.kramerius.security.licenses.LicensesManager; -import cz.incad.kramerius.security.licenses.LicensesManagerException; - - -public class DisplayRightView extends AbstractRightsView { - - public static final String ACTION="action"; - - @Inject - UserManager userManager; - - @Inject - Provider userProvider; - - - @Inject - RightCriteriumWrapperFactory factory; - - @Inject - RightsManager rightsManager; - - @Inject - LicensesManager labelsManager; - - Right right; - - - public String getAppliedRole() throws RecognitionException, TokenStreamException { - if (getAction() == Actions.create) { - return "common_users"; - } else { - Right right = getRight(); - //TODO: change it !! - Role role = (Role) right.getRole(); - return role.getName(); - } - } - - public Right getRight() throws RecognitionException, TokenStreamException { - if ((this.right == null) && (getRightIdParam() != null)){ - this.right = this.rightsManager.findRightById(Integer.parseInt(getRightIdParam())); - } - return this.right; - } - - public String[] getRoles() { - User user = this.userProvider.get(); - Role[] roles = null; - - if (hasSuperAdminRole(this.userProvider.get())) { - roles = userManager.findAllRoles(""); - } else { - int[] grps = getUserGroups(user); - roles = userManager.findAllRoles(grps, ""); - } - return getRoleNames(roles); - } - - - public String[] getRoleNames(Role[] roles) { - String[] strRoles = new String[roles.length]; - for (int i = 0; i < strRoles.length; i++) { - String rname = roles[i].getName(); - rname = rname.replace("\"", "\\\""); - strRoles[i] = rname; - } - return strRoles; - } - - - public int[] getUserGroups(User user) { - Role[] grps = user.getGroups(); - int[] grpIds = new int[grps.length]; - for (int i = 0; i < grpIds.length; i++) { - grpIds[i] = grps[i].getId(); - } - return grpIds; - } - - - public int getCritparamsid() throws RecognitionException, TokenStreamException { - RightCriteriumParams params = getParams(); - return params != null ? params.getId() : 0; - } - - - public int getPriority() throws RecognitionException, TokenStreamException { - Right r = getRight(); - if (r != null ) { - return r.getFixedPriority(); - } else return 0; - } - - public RightCriteriumParams getParams() throws RecognitionException, TokenStreamException { - Right r = getRight(); - if (r != null && r.getCriteriumWrapper() != null && r.getCriteriumWrapper().getCriteriumParams() != null) { - return r.getCriteriumWrapper().getCriteriumParams(); - } else return null; - - } - - public String getCritparamdesc() throws RecognitionException, TokenStreamException { - RightCriteriumParams params = getParams(); - return params != null ? params.getShortDescription() : ""; - } - - public Object[] getCritparams() throws RecognitionException, TokenStreamException { - RightCriteriumParams params = getParams(); - if (params != null) { - List retvals = new ArrayList<>(); - Object[] objs = params.getObjects(); - for (Object o : - objs) { - if (o instanceof String) { - String str = (String) o; - // rendering to javascript '\' must be escaped - // replacing what = \ - StringBuilder what = new StringBuilder(); - what.append("\\\\"); - // replacing by = \\ - StringBuilder byWhat = new StringBuilder(); - byWhat.append("\\\\\\\\"); - - String s = str.replaceAll(what.toString(), byWhat.toString()); - retvals.add(s); - } else { - retvals.add(o); - } - - } - return retvals.toArray(); - - } else return new Object[0]; - } - - public String getCriterium() throws RecognitionException, TokenStreamException { - Right r = getRight(); - if (r != null) { - return r.getCriteriumWrapper() != null ? r.getCriteriumWrapper().getRightCriterium().getQName() : ""; - } else return ""; - } - - public List getCriteriums() throws TokenStreamException, RecognitionException { - List pidsParams = getPidsParams(); - boolean found = pidsParams.stream().anyMatch((v) -> v.toString().equals(SpecialObjects.REPOSITORY.getPid())); - List criteriums = factory.createAllCriteriumWrappers(SecuredActions.findByFormalName(getSecuredAction())); - if (!found || pidsParams.size() > 1) { - return criteriums.stream().filter(crit -> !crit.getRightCriterium().isRootLevelCriterum()).collect(Collectors.toList()); - } else { - return criteriums; - } - } - - public List getRightCriteriumParams() { - RightCriteriumParams[] allParams = this.rightsManager.findAllParams(); - return Arrays.asList(allParams); - } - - - public List getLabels() throws LicensesManagerException { - return this.labelsManager.getLabels(); - } - - public Actions getAction() { - String parameter = this.requestProvider.get().getParameter(ACTION); - if (parameter == null) return Actions.create; - else return Actions.valueOf(parameter); - } - - - public boolean isJustCreated() { - return getAction() == Actions.create; - } - - - - public static enum Actions { - edit { - - @Override - public String getName() { - return name(); - } - }, - create { - - @Override - public String getName() { - return name(); - } - - }; - - public abstract String getName(); - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/rights/DisplayRightsForObjectsView.java b/search/src/java/cz/incad/Kramerius/views/rights/DisplayRightsForObjectsView.java deleted file mode 100644 index 52ead72340..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/rights/DisplayRightsForObjectsView.java +++ /dev/null @@ -1,331 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views.rights; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.logging.Level; - -import javax.inject.Named; - -import antlr.RecognitionException; -import antlr.TokenStreamException; - -import com.google.inject.Inject; -import com.google.inject.Provider; - -import cz.incad.Kramerius.security.strenderers.RightWrapper; -import cz.incad.Kramerius.security.strenderers.TitlesForObjects; -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.ObjectPidsPath; -import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.security.RightsResolver; -import cz.incad.kramerius.security.Right; -import cz.incad.kramerius.security.RightsManager; -import cz.incad.kramerius.security.SecuredActions; -import cz.incad.kramerius.security.SpecialObjects; -import cz.incad.kramerius.security.User; -import cz.incad.kramerius.security.UserManager; -import cz.incad.kramerius.security.*; -import cz.incad.kramerius.security.utils.SortingRightsUtils; -import cz.incad.kramerius.service.ResourceBundleService; -import cz.incad.kramerius.utils.pid.LexerException; -import cz.incad.kramerius.utils.pid.PIDParser; -import cz.incad.kramerius.virtualcollections.CollectionException; -import cz.incad.kramerius.virtualcollections.CollectionsManager; - -public class DisplayRightsForObjectsView extends AbstractRightsView { - - private static final String DEFAULT_NAME_INSTANCE = "default"; - - static java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(DisplayRightsForObjectsView.class.getName()); - - @Inject - @Named("securedFedoraAccess") - FedoraAccess fedoraAccess; - - @Inject - RightsManager rightsManager; - - @Inject - RightsResolver rightsResolver; - - @Inject - Provider userProvider; - - @Inject - @Named("new-index") - SolrAccess solrAccess; - - @Inject - UserManager userManager; - - @Inject - Provider localesProvider; - - @Inject - ResourceBundleService resourceBundleService; - - @Inject - @Named("fedora") - CollectionsManager collectionGet; - - public DisplayRightsForObjectsView() { - super(); - } - - - public Right[] allRights(ObjectPidsPath path) { - return rightsManager.findAllRights(path.getPathFromRootToLeaf(), getSecuredAction()); - } - - - public String getRequestedStream() { - try { - String retVal = null; - List params = getPidsParams(); - if (!params.isEmpty()) { - PIDParser pidParser = new PIDParser(params.get(0).toString()); - pidParser.objectPid(); - retVal = pidParser.isDatastreamPid() ? pidParser.getDataStream() : DEFAULT_NAME_INSTANCE; - } else { - retVal = DEFAULT_NAME_INSTANCE; - } - return retVal; - } catch (RecognitionException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - return DEFAULT_NAME_INSTANCE; - } catch (TokenStreamException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - return DEFAULT_NAME_INSTANCE; - } catch (LexerException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - return DEFAULT_NAME_INSTANCE; - } - - } - - public List getRightsPath() throws CollectionException { - try { - List params = getPidsParams(); - List rightsForPaths = new ArrayList(); - - for (Object pid : params) { - ObjectPidsPath[] paths = solrAccess.getPidPaths(pid.toString()); - - for (ObjectPidsPath path : paths) { - path = path.injectRepository().injectCollections(this.collectionGet); - List pathRights = new ArrayList(Arrays.asList(allRights(path))); - Map map = new HashMap(); - - // ma superadmin roli ? - if (!hasSuperAdminRole(this.userProvider.get())) { - RightsReturnObject[] actionAllowedForAllPath = rightsResolver.isActionAllowedForAllPath(SecuredActions.ADMINISTRATE.getFormalName(), pid.toString(), null, path); - boolean[] booleans = new boolean[actionAllowedForAllPath.length]; - for (int i = 0; i < booleans.length; i++) { - booleans[i]=actionAllowedForAllPath[i].flag(); - } - for (int i = 0; i < booleans.length; i++) { - // can administrate - if (booleans[i]) { - - for (Right right : pathRights) { - if (pid.equals(right.getPid())) { - map.put(right.getId(), true); - } - } - - } else { - for (Right right : pathRights) { - if (pid.equals(right.getPid())) { - map.put(right.getId(), false); - } - } - } - } - } else { - - for (Right right : pathRights) { - map.put(right.getId(), true); - } - } - - Right[] resultRights = SortingRightsUtils.sortRights(pathRights.toArray(new Right[pathRights.size()]), path); - DialogContent dialogContent = new DialogContent(this.fedoraAccess, path, resultRights, this.resourceBundleService, this.localesProvider.get()); - dialogContent.setRightsAccess(map); - rightsForPaths.add(dialogContent); - } - } - - - return rightsForPaths; - - } catch (RecognitionException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } catch (TokenStreamException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } catch (IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } - return new ArrayList(); - } - - - - // reprezentuje jeden dialog prav. Kolik vybranych objektu, tolik dialogu - - public static class DialogContent { - - static java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(DisplayRightsForObjectsView.DialogContent.class.getName()); - // cesta k vybranemu objektu - private ObjectPidsPath path; - - // prava - private RightWrapper[] wrappers; - private Map rightsAccess; - - - private HashMap titles; - private HashMap models; - - - - // muze administrovat vybrany objekt - private boolean canAdministrate; - - public DialogContent(FedoraAccess fedoraAccess, ObjectPidsPath path, Right[] rights, ResourceBundleService resourceBundleService, Locale locale) { - super(); - try { - this.path = path; - this.wrappers = RightWrapper.wrapRights(fedoraAccess, rights); - int length = path.getLength(); - String curPid = path.getNodeFromRootToLeaf(length-1); - - PIDParser pidParser = new PIDParser(curPid); - pidParser.objectPid(); - - - for (RightWrapper rw : this.wrappers) { - rw.setEditable(rw.getPid().equals(curPid)); - } - - - this.titles = TitlesForObjects.createTitlesForPaths(fedoraAccess, this.path); - this.models = TitlesForObjects.createModelsForPaths(fedoraAccess, this.path, resourceBundleService, locale); - - } catch (IOException e) { - throw new RuntimeException(e.getMessage()); - } catch (LexerException e) { - throw new RuntimeException(e.getMessage()); - } - } - - public String getRowId() { - int hashCode = Arrays.hashCode(path.getPathFromRootToLeaf()); - return Integer.toHexString(hashCode); - } - - - - public ObjectPidsPath getPath() { - return path; - } - - public Right[] getRights() { - return wrappers; - } - - - - public Map getTitles() { - return this.titles; - } - - public String getTooltipForPath() { - String[] pathFromRootToLeaf = this.path.getPathFromRootToLeaf(); - StringBuilder builder = new StringBuilder(); - for (int i = 0,ll= pathFromRootToLeaf.length; i < ll; i++) { - builder.append(this.titles.get(pathFromRootToLeaf[i])); - if (SpecialObjects.findSpecialObject(pathFromRootToLeaf[i]) ==null) { - builder.append("(").append(this.models.get(pathFromRootToLeaf[i])).append(")"); - } - if (i < ll -1) { - builder.append("::"); - } - } - return builder.toString(); - } - - - public String getTitleForPath() { - String[] pathFromRootToLeaf = this.path.getPathFromRootToLeaf(); - StringBuilder builder = new StringBuilder(); - for (int i = 0,ll= pathFromRootToLeaf.length; i < ll; i++) { - builder.append(this.titles.get(pathFromRootToLeaf[i])); - if (i < ll -1) { - builder.append("::"); - } - } - return builder.toString(); - } - - - public HashMap getModels() { - return models; - } - - public boolean isCanAdministrate() { - return canAdministrate; - } - - public void setCanAdministrate(boolean canAdministrate) { - this.canAdministrate = canAdministrate; - } - - public Map getRightsAccess() { - return rightsAccess; - } - - public void setRightsAccess(Map rightsAccess) { - this.rightsAccess = rightsAccess; - } - } - - public static class RightContainer { - private String currentPID; - private Right right; - private RightContainer(String currentPID, Right right) { - super(); - this.currentPID = currentPID; - this.right = right; - } - - public Right getRight() { - return right; - } - - public boolean isEditable() { - return this.right.getPid().equals(this.currentPID); - } - - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/rights/DisplaySecuredStreamsView.java b/search/src/java/cz/incad/Kramerius/views/rights/DisplaySecuredStreamsView.java deleted file mode 100644 index 1d32ae96f2..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/rights/DisplaySecuredStreamsView.java +++ /dev/null @@ -1,80 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views.rights; - -import java.util.ArrayList; -import java.util.List; - -import javax.servlet.http.HttpServletRequest; - -import org.antlr.stringtemplate.StringTemplate; - -import antlr.RecognitionException; -import antlr.TokenStreamException; - -import com.google.inject.Inject; -import com.google.inject.Provider; - -import cz.incad.kramerius.utils.conf.KConfiguration; - -public class DisplaySecuredStreamsView extends AbstractRightsView { - - @Inject - KConfiguration configuration; - - @Inject - Provider requestProvider; - - - public List getSecuredStreams() throws RecognitionException, TokenStreamException { - List streamsAndPids = new ArrayList(); - String[] streams = configuration.getSecuredAditionalStreams(); - List pidsParams = getPidsParams(); - for (String strm : streams) { - streamsAndPids.add(new StreamAndPid(pidsParams , strm)); - } - return streamsAndPids; - } - - public static class StreamAndPid { - - private List pids; - - private String stream; - - public StreamAndPid(List pids, String stream) { - super(); - this.pids = pids; - this.stream = stream; - } - - public List getPids() { - return pids; - } - - public String getStreamName() { - return stream; - } - - public String getPidStructsArguments() { - StringTemplate tmpl = new StringTemplate("[$pids:{pid| { pid: '$pid$/$stream$' } };separator=\",\"$]"); - tmpl.setAttribute("pids", this.pids); - tmpl.setAttribute("stream", this.stream); - return tmpl.toString(); - } - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/rights/DisplaySelectionRightsViewObject.java b/search/src/java/cz/incad/Kramerius/views/rights/DisplaySelectionRightsViewObject.java deleted file mode 100644 index 85b4ee1c19..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/rights/DisplaySelectionRightsViewObject.java +++ /dev/null @@ -1,84 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views.rights; - -import java.util.List; -import java.util.logging.Level; - -import javax.inject.Named; - -import antlr.RecognitionException; -import antlr.TokenStreamException; - -import com.google.inject.Inject; -import com.google.inject.Provider; - -import cz.incad.Kramerius.security.strenderers.RightWrapper; -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.security.RightsResolver; -import cz.incad.kramerius.security.Right; -import cz.incad.kramerius.security.RightCriteriumWrapperFactory; -import cz.incad.kramerius.security.RightsManager; -import cz.incad.kramerius.security.User; -import cz.incad.kramerius.security.UserManager; - -public class DisplaySelectionRightsViewObject extends AbstractRightsView { - static java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(DisplayRightsForObjectsView.class.getName()); - - @Inject - @Named("securedFedoraAccess") - FedoraAccess fedoraAccess; - - @Inject - RightsManager rightsManager; - - @Inject - RightsResolver rightsResolver; - - @Inject - Provider userProvider; - - @Inject - SolrAccess solrAccess; - - @Inject - UserManager userManager; - - @Inject - RightCriteriumWrapperFactory factory; - - - public RightWrapper[] getRights() { - try { - List params = getPidsParams(); - String securedAction = getSecuredAction(); - String[] pids = (String[]) params.toArray(new String[params.size()]); - Right[] foundrights = this.rightsManager.findAllRights(pids, securedAction); - RightWrapper[] wrappers = RightWrapper.wrapRights(this.fedoraAccess, foundrights); - return wrappers; - } catch (RecognitionException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } catch (TokenStreamException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } - return new RightWrapper[0]; - } - - - -} diff --git a/search/src/java/cz/incad/Kramerius/views/rights/GlobalActionsView.java b/search/src/java/cz/incad/Kramerius/views/rights/GlobalActionsView.java deleted file mode 100644 index 50b1f772ab..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/rights/GlobalActionsView.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views.rights; - -import java.io.IOException; -import java.util.Locale; -import java.util.ResourceBundle; -import java.util.logging.Level; - -import com.google.inject.Inject; -import com.google.inject.Provider; - -import cz.incad.Kramerius.security.strenderers.SecuredActionWrapper; -import cz.incad.kramerius.security.SecuredActions; -import cz.incad.kramerius.service.ResourceBundleService; - -public class GlobalActionsView { - - static java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(GlobalActionsView.class.getName()); - - @Inject - ResourceBundleService bundleService; - - @Inject - Provider localesProvider; - - - public SecuredActionWrapper[] getWrappers() { - try { - Locale locale = this.localesProvider.get(); - ResourceBundle resbundle = bundleService.getResourceBundle("labels", locale); - return SecuredActionWrapper.wrap(resbundle, SecuredActions.values()); - } catch (Exception e) { - LOGGER.log(Level.SEVERE, e.getMessage(),e); - throw new RuntimeException(e); - } - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/rights/LabelEditView.java b/search/src/java/cz/incad/Kramerius/views/rights/LabelEditView.java deleted file mode 100644 index 82d6dbf6e1..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/rights/LabelEditView.java +++ /dev/null @@ -1,48 +0,0 @@ -package cz.incad.Kramerius.views.rights; - -import com.google.inject.Inject; -import cz.incad.kramerius.Initializable; -import cz.incad.kramerius.security.licenses.License; -import cz.incad.kramerius.security.licenses.LicensesManager; -import cz.incad.kramerius.security.licenses.LicensesManagerException; - -import java.util.logging.Level; -import java.util.logging.Logger; - -public class LabelEditView extends AbstractRightsView implements Initializable { - - public static final Logger LOGGER = Logger.getLogger(LabelEditView.class.getName()); - - public static final String LABEL_ID_PARAM = "id"; - - @Inject - LicensesManager labelsManager; - - License label; - - @Override - public void init() { - try { - String roleId = this.requestProvider.get().getParameter(LABEL_ID_PARAM); - if (roleId != null) { - this.label = this.labelsManager.getLabelById(Integer.parseInt(roleId)); - } - } catch (LicensesManagerException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } - } - - - public int getId() { - return this.label != null ? label.getId() : -1; - } - - public String getName() { - return label != null ? label.getName() : ""; - } - - public String getDescription() { - return this.label != null ? label.getDescription() : ""; - } - -} diff --git a/search/src/java/cz/incad/Kramerius/views/rights/LabelsManageView.java b/search/src/java/cz/incad/Kramerius/views/rights/LabelsManageView.java deleted file mode 100644 index eb5b9184d7..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/rights/LabelsManageView.java +++ /dev/null @@ -1,131 +0,0 @@ -package cz.incad.Kramerius.views.rights; - -import com.google.inject.Inject; -import com.google.inject.Provider; -import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.security.RightsManager; -import cz.incad.kramerius.security.licenses.License; -import cz.incad.kramerius.security.licenses.LicensesManager; -import cz.incad.kramerius.security.licenses.LicensesManagerException; -import cz.incad.kramerius.utils.XMLUtils; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -import javax.inject.Named; -import javax.servlet.http.HttpServletRequest; -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -public class LabelsManageView extends AbstractRightsView { - - public static Logger LOGGER = Logger.getLogger(LabelsManageView.class.getName()); - - @Inject - LicensesManager licensesManager; - - @Inject - RightsManager rightsManager; - - @Inject - Provider requestProvider; - - - @Inject - @Named("new-index") - SolrAccess solrAccess; - - - public List getLabels() { - try { - - Document request = this.solrAccess.requestWithSelectReturningXml("facet.field=licenses&fl=licenses&q=*%3A*&rows=0&facet=on"); - - Element dnntLabelsFromSolr = XMLUtils.findElement(request.getDocumentElement(), new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - String name = element.getAttribute("name"); - return name != null && name.equals("licenses"); - } - }); - - List labelsUsedInSolr = XMLUtils.getElements(dnntLabelsFromSolr).stream().map(element -> { - return element.getAttribute("name"); - }).collect(Collectors.toList()); - - - return this.licensesManager.getLabels().stream().map(label-> { - List> objects = rightsManager.findObjectUsingLabel(label.getId()); - return new LabelListItemView(objects, labelsUsedInSolr, label); - }).collect(Collectors.toList()); - } catch (LicensesManagerException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - return new ArrayList<>(); - } catch (IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - return new ArrayList<>(); - } - } - - - public class LabelListItemView { - - private License license; - private List> objects = null; - private List solrLabels; - - public LabelListItemView(List> rightObjects, List solrLabels, License license) { - this.objects = rightObjects; - this.license = license; - this.solrLabels = solrLabels; - } - - - public List getUsedPids() { - return this.objects.stream().map(item-> {return item.get("pid");}).collect(Collectors.toList()); - } - - public boolean isUsedInRights() { - return !getUsedPids().isEmpty(); - } - - public List getSolrLabels() { - return solrLabels; - } - - - public boolean isUsedInSolr() { - Optional any = getSolrLabels().stream().filter(label -> label.equals(getName())).findAny(); - return any.isPresent(); - } - - public int getId() { - return this.license.getId(); - } - - public String getName() { - return this.license.getName(); - } - - public String getDescription() { - return this.license.getDescription(); - } - - public int getPriority() { - return this.license.getPriority(); - } - - public String getGroup() { - return this.license.getGroup(); - } - - public boolean isLocal() { - return this.license.getGroup() != null && this.license.getGroup().trim().equals(LicensesManager.LOCAL_GROUP_NAME); - } - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/rights/RolesView.java b/search/src/java/cz/incad/Kramerius/views/rights/RolesView.java deleted file mode 100644 index f661d5cc29..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/rights/RolesView.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views.rights; - -import java.util.Arrays; -import java.util.List; - -import javax.servlet.http.HttpServletRequest; - -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.lizardtech.djvu.anno.Mapper; - -import cz.incad.Kramerius.security.strenderers.RoleWrapper; -import cz.incad.kramerius.Initializable; -import cz.incad.kramerius.security.DefaultRoles; -import cz.incad.kramerius.security.RightsManager; -import cz.incad.kramerius.security.Role; -import cz.incad.kramerius.security.User; -import cz.incad.kramerius.security.UserManager; -import cz.incad.utils.K4Collections; -import cz.incad.utils.K4Collections.Combinator; - -public class RolesView extends AbstractRightsView implements Initializable { - - private static final String ROLENAME_PARAM = "rolename"; - - static java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(RolesView.class.getName()); - - @Inject - UserManager userManager; - - @Inject - Provider userProvider; - - @Inject - RightsManager rightsManager; - - @Inject - Provider requestProvider; - - private Role role; - - - - @Override - public void init() { - String rname = this.requestProvider.get().getParameter(ROLENAME_PARAM); - if (rname != null) { - this.role = this.userManager.findRoleByName(rname); - } - } - - - public List getRolesWithNull() { - - List retList; - User user = userProvider.get(); - if (super.hasSuperAdminRole(user)) { - retList = RoleWrapper.wrap(Arrays.asList(userManager.findAllRoles("")),true); - } else { - Role[] roles = user.getGroups(); - int[] roleIds = new int[roles.length]; - for (int i = 0; i < roleIds.length; i++) { - roleIds[i] = roles[i].getId(); - } - retList = RoleWrapper.wrap(Arrays.asList(userManager.findAllRoles(roleIds,"")),true); - } - retList = saturateAdministratorRoles(retList); - return saturateFlags(retList); - } - - - - private List saturateAdministratorRoles(List retList) { - List allList = RoleWrapper.wrap(Arrays.asList(userManager.findAllRoles("")),true); - for (RoleWrapper roleWrapper : retList) { - final int personalAdminId = roleWrapper.getPersonalAdminId(); - Combinator combinator = new Combinator() { - - @Override - public RoleWrapper process(RoleWrapper base, RoleWrapper object) { - if (base != null) return base; - else if (object.getId() == personalAdminId) return object; - return null; - } - }; - RoleWrapper admin = K4Collections.reduce(combinator, null, allList); - roleWrapper.setRoleAdministrator(admin); - } - - return retList; - } - - - public List getRoles() { - List retList; - User user = userProvider.get(); - if (super.hasSuperAdminRole(user)) { - retList = RoleWrapper.wrap(Arrays.asList(userManager.findAllRoles("")),false); - } else { - Role[] roles = user.getGroups(); - int[] roleIds = new int[roles.length]; - for (int i = 0; i < roleIds.length; i++) { - roleIds[i] = roles[i].getId(); - } - retList = RoleWrapper.wrap(Arrays.asList(userManager.findAllRoles(roleIds,"")),false); - } - retList = saturateAdministratorRoles(retList); - return saturateFlags(retList); - } - - public int getRoleId() { - return this.role != null ? this.role.getId() : -1; - } - - public String getRoleName() { - return this.role != null ? this.role.getName() : ""; - } - - - public List saturateFlags(List retList) { - final int[] usedRoles = this.rightsManager.findUsedRoleIDs(); - Arrays.sort(usedRoles); - List rwraps = K4Collections.map(retList, - new K4Collections.Mapper() { - @Override - public RoleWrapper process(RoleWrapper t, int index) { - if (DefaultRoles.findByName(t.getName()) != null) { - t.setCanbedeleted(false); - } else { - int indx = Arrays.binarySearch(usedRoles, t.getId()); - boolean canbedeleted = indx < 0; - t.setCanbedeleted(canbedeleted); - } - return t; - } - }); - - - if (this.role != null) { - final int personalAdminId = role.getPersonalAdminId(); - if (personalAdminId >= 0) { - return K4Collections.map(rwraps, - new K4Collections.Mapper() { - @Override - public RoleWrapper process(RoleWrapper t, int index) { - if (t.getId()== personalAdminId) { - t.setSelected(true); - } - return t; - } - }); - } else { - return rwraps; - } - - } else { - return rwraps; - } - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/rights/UsersView.java b/search/src/java/cz/incad/Kramerius/views/rights/UsersView.java deleted file mode 100644 index dc32968acc..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/rights/UsersView.java +++ /dev/null @@ -1,182 +0,0 @@ -/* - * Copyright (C) 2012 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -/** - * - */ -package cz.incad.Kramerius.views.rights; - -import java.util.Arrays; -import java.util.List; - -import javax.servlet.http.HttpServletRequest; - -import com.google.inject.Inject; -import com.google.inject.Provider; - -import cz.incad.Kramerius.security.strenderers.RoleWrapper; -import cz.incad.kramerius.Initializable; -import cz.incad.kramerius.security.DefaultRoles; -import cz.incad.kramerius.security.RightsManager; -import cz.incad.kramerius.security.Role; -import cz.incad.kramerius.security.User; -import cz.incad.kramerius.security.UserManager; -import cz.incad.utils.K4Collections; -import cz.incad.utils.K4Collections.Combinator; - -/** - * @author pavels - * - */ -public class UsersView extends AbstractRightsView implements Initializable { - - private static final String ROLENAME_PARAM = "rolename"; - - static java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(RolesView.class.getName()); - - @Inject - UserManager userManager; - - @Inject - Provider userProvider; - - @Inject - RightsManager rightsManager; - - @Inject - Provider requestProvider; - - private Role role; - - - - @Override - public void init() { - String rname = this.requestProvider.get().getParameter(ROLENAME_PARAM); - if (rname != null) { - this.role = this.userManager.findRoleByName(rname); - } - } - - - public List getRolesWithNull() { - - List retList; - User user = userProvider.get(); - if (super.hasSuperAdminRole(user)) { - retList = RoleWrapper.wrap(Arrays.asList(userManager.findAllRoles("")),true); - } else { - Role[] roles = user.getGroups(); - int[] roleIds = new int[roles.length]; - for (int i = 0; i < roleIds.length; i++) { - roleIds[i] = roles[i].getId(); - } - retList = RoleWrapper.wrap(Arrays.asList(userManager.findAllRoles(roleIds,"")),true); - } - retList = saturateAdministratorRoles(retList); - return saturateFlags(retList); - } - - - - private List saturateAdministratorRoles(List retList) { - List allList = RoleWrapper.wrap(Arrays.asList(userManager.findAllRoles("")),true); - for (RoleWrapper roleWrapper : retList) { - final int personalAdminId = roleWrapper.getPersonalAdminId(); - Combinator combinator = new Combinator() { - - @Override - public RoleWrapper process(RoleWrapper base, RoleWrapper object) { - if (base != null) return base; - else if (object.getId() == personalAdminId) return object; - return null; - } - }; - RoleWrapper admin = K4Collections.reduce(combinator, null, allList); - roleWrapper.setRoleAdministrator(admin); - } - - return retList; - } - - - public List getRoles() { - List retList; - User user = userProvider.get(); - if (super.hasSuperAdminRole(user)) { - retList = RoleWrapper.wrap(Arrays.asList(userManager.findAllRoles("")),false); - } else { - Role[] roles = user.getGroups(); - int[] roleIds = new int[roles.length]; - for (int i = 0; i < roleIds.length; i++) { - roleIds[i] = roles[i].getId(); - } - retList = RoleWrapper.wrap(Arrays.asList(userManager.findAllRoles(roleIds,"")),false); - } - retList = saturateAdministratorRoles(retList); - return saturateFlags(retList); - } - - public int getRoleId() { - return this.role != null ? this.role.getId() : -1; - } - - public String getRoleName() { - return this.role != null ? this.role.getName() : ""; - } - - - public List saturateFlags(List retList) { - final int[] usedRoles = this.rightsManager.findUsedRoleIDs(); - Arrays.sort(usedRoles); - List rwraps = K4Collections.map(retList, - new K4Collections.Mapper() { - @Override - public RoleWrapper process(RoleWrapper t, int index) { - if (DefaultRoles.findByName(t.getName()) != null) { - t.setCanbedeleted(false); - } else { - int indx = Arrays.binarySearch(usedRoles, t.getId()); - boolean canbedeleted = indx < 0; - t.setCanbedeleted(canbedeleted); - } - return t; - } - }); - - - if (this.role != null) { - final int personalAdminId = role.getPersonalAdminId(); - if (personalAdminId >= 0) { - return K4Collections.map(rwraps, - new K4Collections.Mapper() { - @Override - public RoleWrapper process(RoleWrapper t, int index) { - if (t.getId()== personalAdminId) { - t.setSelected(true); - } - return t; - } - }); - } else { - return rwraps; - } - - } else { - return rwraps; - } - } -} \ No newline at end of file diff --git a/search/src/java/cz/incad/Kramerius/views/statistics/AbstractStatisticsViewObject.java b/search/src/java/cz/incad/Kramerius/views/statistics/AbstractStatisticsViewObject.java deleted file mode 100644 index b1e0bc9514..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/statistics/AbstractStatisticsViewObject.java +++ /dev/null @@ -1,232 +0,0 @@ -/* - * Copyright (C) 2012 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -/** - * - */ -package cz.incad.Kramerius.views.statistics; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.ResourceBundle; -import java.util.Set; - -import javax.servlet.http.HttpServletRequest; - -import com.google.inject.name.Named; -import org.apache.log4j.Logger; - -import com.google.inject.Inject; -import com.google.inject.Provider; - -import cz.incad.Kramerius.statistics.StatisticsExportServlet; -import cz.incad.Kramerius.utils.JSONUtils; -import cz.incad.kramerius.service.ResourceBundleService; -import cz.incad.kramerius.statistics.ReportedAction; -import cz.incad.kramerius.statistics.StatisticReport; -import cz.incad.kramerius.statistics.StatisticsAccessLog; -import cz.incad.kramerius.statistics.StatisticsReportException; -import cz.incad.kramerius.statistics.filters.DateFilter; -import cz.incad.kramerius.statistics.filters.IPAddressFilter; -import cz.incad.kramerius.statistics.filters.ModelFilter; -import cz.incad.kramerius.statistics.filters.PidsFilter; -import cz.incad.kramerius.statistics.filters.StatisticsFilter; -import cz.incad.kramerius.statistics.filters.StatisticsFiltersContainer; -import cz.incad.kramerius.statistics.filters.UniqueIPAddressesFilter; -import cz.incad.kramerius.statistics.filters.VisibilityFilter; -import cz.incad.kramerius.statistics.filters.VisibilityFilter.VisbilityType; -import cz.incad.kramerius.statistics.impl.ModelStatisticReport; -import cz.incad.kramerius.utils.database.Offset; - -/** - * @author pavels - * - */ -public abstract class AbstractStatisticsViewObject { - - public static final Logger LOGGER = Logger.getLogger(AbstractStatisticsViewObject.class.getName()); - static final int MAX_TITLE_LIMIT = 18; - - @Inject - @Named("database") - StatisticsAccessLog statisticsAccessLog; - - @Inject - Provider servletRequestProvider; - - @Inject - Provider localesProvider; - - @Inject - ResourceBundleService resService; - - protected List> data = null; - - public int getMaxValue() throws StatisticsReportException { - int max = -1; - List> report = getReport(); - for (Map map : report) { - boolean containsCount = map.containsKey("count"); - if (containsCount) { - Integer val = (Integer) map.get("count"); - if (max < val) { - max = val; - } - } - } - return max; - } - - public VisibilityFilter getVisibilityFilter() throws IOException { - HttpServletRequest request = this.servletRequestProvider.get(); - String visibility = request.getParameter("visibility"); - VisibilityFilter filter = new VisibilityFilter(); - filter.setSelected(VisibilityFilter.VisbilityType.valueOf(visibility.toUpperCase())); - return filter; - } - - public DateFilter getDateFilter() throws IOException { - HttpServletRequest request = this.servletRequestProvider.get(); - String dFrom = request.getParameter(StatisticsExportServlet.DATE_FROM_ATTRIBUTE); - String dTo = request.getParameter( StatisticsExportServlet.DATE_TO_ATTRIBUTE); - DateFilter dFilter = new DateFilter(); - if (dFrom != null && (!dFrom.trim().equals(""))) { - dFilter.setFromDate(dFrom); - } - if (dTo != null && (!dTo.trim().equals(""))) { - dFilter.setToDate(dTo); - } - return dFilter; - } - - public VisibilityFilter getVisbilityFilter() { - HttpServletRequest request = this.servletRequestProvider.get(); - String vis = request.getParameter(StatisticsExportServlet.VISIBILITY_ATTRIBUTE); - if (vis != null) vis = vis.toUpperCase(); - VisibilityFilter filter = new VisibilityFilter(); - filter.setSelected(VisbilityType.valueOf(vis)); - return filter; - } - - public synchronized List> getReport() throws StatisticsReportException { - try { - if (this.data == null) { - HttpServletRequest request = this.servletRequestProvider.get(); - String type = request.getParameter("type"); - String val = request.getParameter("val"); - String ip = request.getParameter("ipaddresses"); - String pids = request.getParameter("pids"); - - String uniqueIP = request.getParameter("uniqueipaddresses"); - - String actionFilter = request.getParameter("action"); - String offset = request.getParameter("offset") != null ? request.getParameter("offset") : "0"; - String size = request.getParameter("size") != null ? request.getParameter("size") : "20"; - - DateFilter dateFilter = getDateFilter(); - ModelFilter modelFilter = new ModelFilter(); - modelFilter.setModel(val); - VisibilityFilter visFilter = getVisbilityFilter(); - - PidsFilter pidsFilter = new PidsFilter(); - pidsFilter.setPids(pids); - - UniqueIPAddressesFilter uniqueIPFilter = new UniqueIPAddressesFilter(); - uniqueIPFilter.setUniqueIPAddressesl(Boolean.valueOf(uniqueIP)); - - IPAddressFilter ipAddr = new IPAddressFilter(); - if (ip != null && !ip.isEmpty()) { - ip = ip.replace(",", "|"); - ip = ip.replace("*", "%"); - ip = ip.replace(" ", ""); - ipAddr.setIpAddress(ip); - } - else { - String ipConfigVal = ipAddr.getValue(); - if (ipConfigVal != null) { - ipConfigVal = ipConfigVal.replace("*", "%"); - } - ipAddr.setIpAddress(ipConfigVal); - } - - StatisticReport report = statisticsAccessLog.getReportById(type); - Offset reportOff = new Offset(offset, size); - report.prepareViews(actionFilter != null ? ReportedAction.valueOf(actionFilter) : null ,new StatisticsFiltersContainer(new StatisticsFilter[] {dateFilter,modelFilter, visFilter, ipAddr, pidsFilter})); - this.data = report.getReportPage(actionFilter != null ? ReportedAction.valueOf(actionFilter) : null ,new StatisticsFiltersContainer(new StatisticsFilter[] {dateFilter,modelFilter, visFilter,ipAddr, uniqueIPFilter, pidsFilter}), reportOff); - } - return this.data; - } catch (IOException e) { - throw new StatisticsReportException(e); - } - } - - public int getPageIndex() { - HttpServletRequest request = this.servletRequestProvider.get(); - String offset = request.getParameter("offset") != null ? request.getParameter("offset") : "0"; - String size = request.getParameter("size") != null ? request.getParameter("size") : "20"; - int sizeInt = Integer.parseInt(size); - int offsetInt = Integer.parseInt(offset); - int pagei = (offsetInt /sizeInt)+1; - return pagei; - } - - - public boolean getDisplayFirstFlag() { - return getPageIndex() > 1; - } - - public boolean getDisplayLastFlag() throws StatisticsReportException { - HttpServletRequest request = this.servletRequestProvider.get(); - String size = request.getParameter("size") != null ? request.getParameter("size") : "20"; - List> report = getReport(); - return report.size() >= Integer.parseInt(size); - } - - public List> getJsonAwareReport() throws StatisticsReportException { - List> newResult = new ArrayList>(); - List> report = getReport(); - for (Map map : report) { - Map jsonedMap = jsoned(map); - newResult.add(jsonedMap); - } - return newResult; - } - - private Map jsoned(Map map) { - Map retval = new HashMap(); - Set keys = map.keySet(); - for (String key : keys) { - Object val = map.get(key); - if (val instanceof String) { - if (key.equals(ModelStatisticReport.TITLE_KEY)) { - if (val.toString().length() > MAX_TITLE_LIMIT) { - val = ((String)val).substring(0, MAX_TITLE_LIMIT)+" ..."; - } - } - retval.put(key, JSONUtils.escaped((String) val)); - } else { - retval.put(key, val); - } - } - return retval; - } - - -} diff --git a/search/src/java/cz/incad/Kramerius/views/statistics/AuthorStatisticsViewObject.java b/search/src/java/cz/incad/Kramerius/views/statistics/AuthorStatisticsViewObject.java deleted file mode 100644 index 75e2b5a276..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/statistics/AuthorStatisticsViewObject.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (C) 2012 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -/** - * - */ -package cz.incad.Kramerius.views.statistics; - -import javax.servlet.http.HttpServletRequest; - -/** - * @author pavels - * - */ -public class AuthorStatisticsViewObject extends AbstractStatisticsViewObject { - - - public String getNext() { - HttpServletRequest request = this.servletRequestProvider.get(); - String offset = request.getParameter("offset") != null ? request.getParameter("offset") : "0"; - String size = request.getParameter("size") != null ? request.getParameter("size") : "20"; - String val = request.getParameter("val"); - int sizeInt = Integer.parseInt(size); - int offsetInt = (Integer.parseInt(offset))+sizeInt; - return "javascript:statistics.reloadAuthorsReport(_action(),_visibility(),$('#report_date_from').val(),$('#report_date_to').val(),'"+val+"',"+offsetInt+","+size+", _ip_address(), _ip_address_unique());"; - } - - public String getPrev() { - HttpServletRequest request = this.servletRequestProvider.get(); - String val = request.getParameter("val"); - String offset = request.getParameter("offset") != null ? request.getParameter("offset") : "0"; - String size = request.getParameter("size") != null ? request.getParameter("size") : "20"; - int sizeInt = Integer.parseInt(size); - int offsetInt = Math.max((Integer.parseInt(offset)-sizeInt), 0); - return "javascript:statistics.reloadAuthorsReport(_action(),_visibility(),$('#report_date_from').val(),$('#report_date_to').val(),'"+val+"',"+offsetInt+","+size+", _ip_address(), _ip_address_unique());"; - } - - public String getGraphTitle() { - return ""; - } - -} diff --git a/search/src/java/cz/incad/Kramerius/views/statistics/LanguageStatisticsViewObject.java b/search/src/java/cz/incad/Kramerius/views/statistics/LanguageStatisticsViewObject.java deleted file mode 100644 index 0ac9283556..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/statistics/LanguageStatisticsViewObject.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright (C) 2012 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -/** - * - */ -package cz.incad.Kramerius.views.statistics; - -import javax.servlet.http.HttpServletRequest; - -/** - * @author pavels - * - */ -public class LanguageStatisticsViewObject extends AbstractStatisticsViewObject { - - - public String getNext() { - HttpServletRequest request = this.servletRequestProvider.get(); - String offset = request.getParameter("offset") != null ? request.getParameter("offset") : "0"; - String size = request.getParameter("size") != null ? request.getParameter("size") : "20"; - String val = request.getParameter("val"); - int sizeInt = Integer.parseInt(size); - int offsetInt = (Integer.parseInt(offset))+sizeInt; - return "javascript:statistics.reloadLangReport(_action(),_visibility(),'"+val+"',"+offsetInt+","+size+",_ip_address(), _ip_address_unique());"; - } - - public String getPrev() { - HttpServletRequest request = this.servletRequestProvider.get(); - String val = request.getParameter("val"); - String offset = request.getParameter("offset") != null ? request.getParameter("offset") : "0"; - String size = request.getParameter("size") != null ? request.getParameter("size") : "20"; - int sizeInt = Integer.parseInt(size); - int offsetInt = Math.max((Integer.parseInt(offset)-sizeInt), 0); - return "javascript:statistics.reloadLangReport(_action(),_visibility(),'"+val+"',"+offsetInt+","+size+",_ip_address(), _ip_address_unique());"; - } - - public String getGraphTitle() { - return ""; - } - -} diff --git a/search/src/java/cz/incad/Kramerius/views/statistics/ModelStatisticsViewObject.java b/search/src/java/cz/incad/Kramerius/views/statistics/ModelStatisticsViewObject.java deleted file mode 100644 index fdf1bf006d..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/statistics/ModelStatisticsViewObject.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright (C) 2012 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -/** - * - */ -package cz.incad.Kramerius.views.statistics; - -import java.io.IOException; -import java.util.*; - -import javax.servlet.http.HttpServletRequest; - -import cz.incad.kramerius.statistics.ReportedAction; -import cz.incad.kramerius.statistics.StatisticReport; -import cz.incad.kramerius.statistics.StatisticsReportException; -import cz.incad.kramerius.statistics.impl.ModelStatisticReport; - -/** - * @author pavels - */ -public class ModelStatisticsViewObject extends AbstractStatisticsViewObject { - - public String getGraphTitle() throws IOException { - return "Graf objektu typu :"+getSelectedModel(); - } - - - public String getAllFilterOption() { - return "All"; - } - - public String[] getActionFilterOptions() { - List names = new ArrayList(); - ReportedAction[] actions = ReportedAction.values(); - for (ReportedAction ract : actions) { - names.add(ract.name()); - } - return (String[]) names.toArray(new String[names.size()]); - } - - public String getSelectedModel() throws IOException { - HttpServletRequest request = this.servletRequestProvider.get(); - String type = request.getParameter("val"); - - ResourceBundle resBundle = resService.getResourceBundle("labels", this.localesProvider.get()); - String str = resBundle.getString("fedora.model."+type); - return str; - } - - - - public String getCurrentYear() { - Calendar calendar = Calendar.getInstance(); - int year = calendar.get(Calendar.YEAR); - return ""+year; - } - - - public String getPreviousYear() { - Calendar calendar = Calendar.getInstance(); - int year = calendar.get(Calendar.YEAR); - return ""+(year-1); - } - - - - public List getModels() { - StatisticReport report = statisticsAccessLog.getReportById(ModelStatisticReport.REPORT_ID); - return report.getOptionalValues(); - } - - - - - public String getPrev() { - HttpServletRequest request = this.servletRequestProvider.get(); - String val = request.getParameter("val"); - String type = request.getParameter("type"); - String offset = request.getParameter("offset") != null ? request.getParameter("offset") : "0"; - String size = request.getParameter("size") != null ? request.getParameter("size") : "20"; - int sizeInt = Integer.parseInt(size); - int offsetInt = Math.max((Integer.parseInt(offset)-sizeInt), 0); - return "javascript:statistics.reloadModelReport(_action(),_visibility(),$('#report_date_from').val(),$('#report_date_to').val(),'"+type+"','"+val+"',"+offsetInt+","+size+", _ip_address(), _ip_address_unique());"; - } - - public String getNext() { - HttpServletRequest request = this.servletRequestProvider.get(); - String offset = request.getParameter("offset") != null ? request.getParameter("offset") : "0"; - String size = request.getParameter("size") != null ? request.getParameter("size") : "20"; - String val = request.getParameter("val"); - String type = request.getParameter("type"); - int sizeInt = Integer.parseInt(size); - int offsetInt = (Integer.parseInt(offset))+sizeInt; - return "javascript:statistics.reloadModelReport(_action(),_visibility(),$('#report_date_from').val(),$('#report_date_to').val(),'"+type+"','"+val+"',"+offsetInt+","+size+",_ip_address(), _ip_address_unique());"; - } - -} diff --git a/search/src/java/cz/incad/Kramerius/views/statistics/PidsStatisticsViewObject.java b/search/src/java/cz/incad/Kramerius/views/statistics/PidsStatisticsViewObject.java deleted file mode 100644 index 91079f0a8e..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/statistics/PidsStatisticsViewObject.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * To change this license header, choose License Headers in Project Properties. - * To change this template file, choose Tools | Templates - * and open the template in the editor. - */ -package cz.incad.Kramerius.views.statistics; - -import javax.servlet.http.HttpServletRequest; - -/** - * - * @author Gabriela Melingerová - */ -public class PidsStatisticsViewObject extends AbstractStatisticsViewObject { - - - public String getNext() { - HttpServletRequest request = this.servletRequestProvider.get(); - String offset = request.getParameter("offset") != null ? request.getParameter("offset") : "0"; - String size = request.getParameter("size") != null ? request.getParameter("size") : "20"; - int sizeInt = Integer.parseInt(size); - int offsetInt = (Integer.parseInt(offset))+sizeInt; - return "javascript:statistics.reloadPidsReport(_action(), _pids(), _visibility(),$('#report_date_from_pid').val(),$('#report_date_to_pid').val(),"+offsetInt+","+size+", _ip_address(), _ip_address_unique());"; - } - - public String getPrev() { - HttpServletRequest request = this.servletRequestProvider.get(); - String offset = request.getParameter("offset") != null ? request.getParameter("offset") : "0"; - String size = request.getParameter("size") != null ? request.getParameter("size") : "20"; - int sizeInt = Integer.parseInt(size); - int offsetInt = Math.max((Integer.parseInt(offset)-sizeInt), 0); - return "javascript:statistics.reloadPidsReport(_action(), _pids(), _visibility(),$('#report_date_from_pid').val(),$('#report_date_to_pid').val(),"+offsetInt+","+size+", _ip_address(), _ip_address_unique());"; - } - - public String getGraphTitle() { - return ""; - } - -} diff --git a/search/src/java/cz/incad/Kramerius/views/statistics/details/ModelStatisticsDetail.java b/search/src/java/cz/incad/Kramerius/views/statistics/details/ModelStatisticsDetail.java deleted file mode 100644 index e1561b90cb..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/statistics/details/ModelStatisticsDetail.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright (C) 2012 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -/** - * - */ -package cz.incad.Kramerius.views.statistics.details; - -import java.io.IOException; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.ResourceBundle; - -import javax.servlet.http.HttpServletRequest; - -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.google.inject.name.Named; - -import cz.incad.Kramerius.utils.JSONUtils; -import cz.incad.Kramerius.views.AbstractPrintViewObject; -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.document.model.DCConent; -import cz.incad.kramerius.document.model.utils.DCContentUtils; -import cz.incad.kramerius.document.model.utils.DescriptionUtils; -import cz.incad.kramerius.service.ResourceBundleService; - -/** - * @author pavels - */ -public class ModelStatisticsDetail { - - @Inject - @Named("securedFedoraAccess") - FedoraAccess fedoraAccess; - - @Inject - @Named("new-index") - SolrAccess solrAccess; - - @Inject - Provider requestProvider; - - @Inject - Provider localesProvider; - - @Inject - ResourceBundleService resourceBundleService; - - - public String getTitle() throws IOException { - ResourceBundle resBundle = resourceBundleService.getResourceBundle("labels", localesProvider.get()); - - String pid = requestProvider.get().getParameter("pid"); - Map> dcs = DCContentUtils.getDCS(fedoraAccess, solrAccess, Arrays.asList(pid)); - - StringBuilder builder = new StringBuilder(); - List contents = dcs.get(pid); - - if (contents == null) { - return builder.toString(); - } - - for (int i = 0,ll=contents.size(); i < ll; i++) { - DCConent dcConent = contents.get(i); - String model = dcConent.getType(); - String i18n = null; - String resBundleKey = "fedora.model."+model; - if (resBundle.containsKey(resBundleKey)) { - i18n = resBundle.getString(model); - } else { - i18n = model; - } - if (i > 0) builder.append(" | "); - builder.append(i18n).append(":").append(JSONUtils.escaped(dcConent.getTitle())); - } - - return builder.toString(); - } - - public String[] getDescriptions() throws IOException { - String pid = requestProvider.get().getParameter("pid"); - Map> dcs = DCContentUtils.getDCS(fedoraAccess, solrAccess, Arrays.asList(pid)); - String[] descriptions = DescriptionUtils.getDescriptions(this.resourceBundleService.getResourceBundle("labels", localesProvider.get()), dcs, Arrays.asList(pid)); - for (int i = 0; i < descriptions.length; i++) { - descriptions[i] = JSONUtils.escaped(descriptions[i]); - } - return descriptions; - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/virtualcollection/CollectionItemViewObject.java b/search/src/java/cz/incad/Kramerius/views/virtualcollection/CollectionItemViewObject.java deleted file mode 100644 index 6a5083dc5a..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/virtualcollection/CollectionItemViewObject.java +++ /dev/null @@ -1,73 +0,0 @@ -package cz.incad.Kramerius.views.virtualcollection; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.imaging.ImageStreams; -import cz.incad.kramerius.virtualcollections.Collection; -import cz.incad.kramerius.virtualcollections.Collection.Description; - -public class CollectionItemViewObject { - - private Collection collection; - private boolean thumbnailAvailable = false; - private boolean fullAvailable = false; - - public CollectionItemViewObject(Collection collection, FedoraAccess fedoraAccess) throws IOException { - super(); - this.collection = collection; - this.thumbnailAvailable = fedoraAccess.isStreamAvailable(this.collection.getPid(), ImageStreams.IMG_THUMB.name()); - this.fullAvailable = fedoraAccess.isStreamAvailable(this.collection.getPid(), ImageStreams.IMG_FULL.name()); - } - - public String getPid() { - return collection.getPid(); - } - - public String getLabel() { - return collection.getLabel(); - } - /** - * @return the canLeave - */ - public boolean isCanLeave() { - return collection.isCanLeaveFlag(); - } - - public int getNumberOfDocs() { - return this.collection.getNumberOfDocs(); - } - - public boolean isThumbnailAvailable() { - return thumbnailAvailable; - } - - public boolean isFullAvailable() { - return fullAvailable; - } - - public List getDescriptions() { - return collection.getDescriptions(); - } - - - public Map getDescriptionsMap(){ - Map map = new HashMap(); - for(Description d : this.collection.getDescriptions()){ - map.put(d.getLangCode(), d.getText()); - } - return map; - } - - public Map getLongDescriptionsMap(){ - Map map = new HashMap(); - for(Description d : this.collection.getDescriptions()){ - map.put(d.getLangCode(), d.getLongText()); - } - return map; - } - -} diff --git a/search/src/java/cz/incad/Kramerius/views/virtualcollection/VirtualCollectionServlet.java b/search/src/java/cz/incad/Kramerius/views/virtualcollection/VirtualCollectionServlet.java deleted file mode 100644 index fce013811b..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/virtualcollection/VirtualCollectionServlet.java +++ /dev/null @@ -1,406 +0,0 @@ -package cz.incad.Kramerius.views.virtualcollection; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.logging.Level; - -import javax.servlet.ServletException; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; - - -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.google.inject.name.Named; - -import biz.sourcecode.base64Coder.Base64Coder; -import cz.incad.Kramerius.backend.guice.GuiceServlet; -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.fedora.om.RepositoryException; -import cz.incad.kramerius.processes.starter.ProcessStarter; -import cz.incad.kramerius.ObjectPidsPath; -import cz.incad.kramerius.processes.utils.ProcessUtils; -import cz.incad.kramerius.security.*; -import cz.incad.kramerius.security.SecurityException; -import cz.incad.kramerius.utils.IOUtils; -import cz.incad.kramerius.utils.conf.KConfiguration; -import cz.incad.kramerius.virtualcollections.Collection; -import cz.incad.kramerius.virtualcollections.CollectionException; -import cz.incad.kramerius.virtualcollections.CollectionUtils; -import cz.incad.kramerius.virtualcollections.CollectionsManager; - -import java.io.PrintWriter; -import java.net.URL; -import java.net.URLEncoder; -import java.util.Enumeration; -import java.util.HashMap; -import java.util.Map; - -import cz.incad.kramerius.virtualcollections.impl.fedora.FedoraCollectionsManagerImpl; -import org.apache.commons.fileupload.FileItemIterator; -import org.apache.commons.fileupload.FileItemStream; -import org.apache.commons.fileupload.disk.DiskFileItemFactory; -import org.apache.commons.fileupload.servlet.ServletFileUpload; - - -/** disabled in cdk instance */ -@Deprecated -public class VirtualCollectionServlet extends GuiceServlet { - - public static final java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(VirtualCollectionServlet.class.getName()); - @Inject - @Named("securedFedoraAccess") - FedoraAccess fedoraAccess; - - @Inject - KConfiguration kConfiguration; - public static final String ACTION_NAME = "action"; - - @Inject - @Named("fedora") - CollectionsManager collectionManager; - - @Inject - RightsResolver rightsResolver; - - @Inject - Provider userProvider; - - - boolean permit(User user) { - if (user != null) - return this.rightsResolver.isActionAllowed(user, SecuredActions.VIRTUALCOLLECTION_MANAGE.getFormalName(), - SpecialObjects.REPOSITORY.getPid(), null, ObjectPidsPath.REPOSITORY_PATH).flag(); - else - return false; - } - - - @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - try { - Actions actionToDo = Actions.CHANGE; - String actionNameParam = req.getParameter(ACTION_NAME); - if (actionNameParam != null) { - actionToDo = Actions.valueOf(actionNameParam); - } - try { - User user = this.userProvider.get(); - if (this.permit(user)) { - actionToDo.doPerform(this, fedoraAccess, this.collectionManager, req, resp); - } else { - resp.setStatus(HttpServletResponse.SC_FORBIDDEN); - } - } catch (IOException e1) { - LOGGER.log(Level.SEVERE, e1.getMessage(), e1); - resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e1.toString()); - PrintWriter out = resp.getWriter(); - out.print(e1.toString()); - } catch (SecurityException e1) { - LOGGER.log(Level.INFO, e1.getMessage()); - resp.setStatus(HttpServletResponse.SC_FORBIDDEN); - } catch (Exception e1) { - LOGGER.log(Level.SEVERE, e1.getMessage(), e1); - resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - PrintWriter out = resp.getWriter(); - resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e1.toString()); - out.print(e1.toString()); - } - } catch (SecurityException e) { - resp.sendError(HttpServletResponse.SC_FORBIDDEN); - } - } - - - - @Override - protected void doPost(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - try { - PostActions actionToDo = PostActions.IMAGES_UPLOAD; - String actionNameParam = req.getParameter(ACTION_NAME); - if (actionNameParam != null) { - User user = this.userProvider.get(); - if (this.permit(user)) { - actionToDo = PostActions.valueOf(actionNameParam); - } else { - resp.setStatus(HttpServletResponse.SC_FORBIDDEN); - } - } - try { - actionToDo.doPerform(this, fedoraAccess, this.collectionManager, req, resp); - } catch (IOException e1) { - LOGGER.log(Level.SEVERE, e1.getMessage(), e1); - resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e1.toString()); - PrintWriter out = resp.getWriter(); - out.print(e1.toString()); - } catch (SecurityException e1) { - LOGGER.log(Level.INFO, e1.getMessage()); - resp.setStatus(HttpServletResponse.SC_FORBIDDEN); - } catch (Exception e1) { - LOGGER.log(Level.SEVERE, e1.getMessage(), e1); - resp.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - PrintWriter out = resp.getWriter(); - resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e1.toString()); - out.print(e1.toString()); - } - } catch (SecurityException e) { - resp.sendError(HttpServletResponse.SC_FORBIDDEN); - } - } - - - - protected void writeOutput(HttpServletRequest req, HttpServletResponse resp, String s) throws IOException { - resp.setCharacterEncoding("UTF-8"); - PrintWriter out = resp.getWriter(); - out.print(s); - } - - protected String[] getLangs() { - String[] langs = kConfiguration.getPropertyList("interface.languages"); - return langs; - } - - enum PostActions { - - /** long text uploading */ - LONG_TEXT_UPLOAD { - - @Override - void doPerform(VirtualCollectionServlet vc, - FedoraAccess fedoraAccess, - CollectionsManager colMan, - HttpServletRequest req, - HttpServletResponse response) throws Exception, SecurityException { - String collection = req.getParameter("pid"); - String language = req.getParameter("lang"); - - String encodedProfile = req.getParameter("encodedData"); - byte[] decoded = Base64Coder.decode(encodedProfile); - - CollectionUtils.modifyLangDatastream(collection, language, FedoraCollectionsManagerImpl.LONG_TEXT_DS_PREFIX +language,new String(decoded, "UTF-8"), fedoraAccess); - } - }, - - /** image uploads */ - IMAGES_UPLOAD { - @Override - void doPerform(VirtualCollectionServlet vc, FedoraAccess fedoraAccess, CollectionsManager colMan, - HttpServletRequest req, HttpServletResponse response) throws Exception, SecurityException { - - String collection = req.getParameter("pid"); - - ServletFileUpload upload = new ServletFileUpload(new DiskFileItemFactory()); - FileItemIterator iterator = upload.getItemIterator(req); - - while (iterator.hasNext()) { - FileItemStream fileItemStream = iterator.next(); - String filename = fileItemStream.getName(); - - if (!fileItemStream.isFormField()) { - InputStream inputStream = fileItemStream.openStream(); - String streamName = fileItemStream.getFieldName(); - String contentType = fileItemStream.getContentType(); - ByteArrayOutputStream bos = new ByteArrayOutputStream(); - IOUtils.copyStreams(inputStream, bos);; - LOGGER.info("Creating stream '"+streamName+"' for collection '"+collection+"'"); - CollectionUtils.modifyManagedDatastream(collection, streamName, contentType, bos.toByteArray(), fedoraAccess); - } else { - throw new IllegalArgumentException("illegal argument! "); - } - } - } - }; - abstract void doPerform(VirtualCollectionServlet vc, FedoraAccess fedoraAccess, CollectionsManager colMan, HttpServletRequest req, HttpServletResponse response) throws Exception, SecurityException; - } - - enum Actions { - - /** - * Request to check if pid is in virtual collection - */ - CHECK { - - @Override - void doPerform(VirtualCollectionServlet vc, FedoraAccess fedoraAccess, CollectionsManager colMan, HttpServletRequest req, HttpServletResponse resp) throws IOException, SecurityException, RepositoryException { - String pid = req.getParameter("pid"); - String collection = req.getParameter("collection"); - resp.setContentType("text/plain"); - if (CollectionUtils.isInCollection(pid, collection, fedoraAccess)) { - vc.writeOutput(req, resp, "1"); - } else { - vc.writeOutput(req, resp, "0"); - } - - } - }, - /** - * Request to add pid to virtual collection - */ - ADDTOCOLLECTIONS { - - @Override - void doPerform(VirtualCollectionServlet vc, FedoraAccess fedoraAccess, CollectionsManager colMan, HttpServletRequest req, HttpServletResponse resp) throws IOException, SecurityException { - String pid = req.getParameter("pid"); - - String base = ProcessUtils.getLrServlet(); - - if (base == null || pid == null) { - LOGGER.severe("Cannot start long running process"); - return; - } - String baseurl = base + "?action=start&def=virtualcollections&out=text"; - - String[] rcollections = req.getParameterValues("remove"); - if (rcollections != null) { - for (String collection : rcollections) { - String token = System.getProperty(ProcessStarter.TOKEN_KEY); - String url = baseurl + "¶ms=remove," + pid + "," + URLEncoder.encode(collection, "UTF-8"); - if(token!=null){ - url += "&token=" + token; - } - ProcessUtils.httpGet(url); - } - } - String[] collections = req.getParameterValues("add"); - if (collections != null) { - for (String collection : collections) { - String token = System.getProperty(ProcessStarter.TOKEN_KEY); - String url = baseurl + "¶ms=add," + pid + "," + collection; - if(token!=null){ - url += "&token=" + token; - } - ProcessUtils.httpGet(url); - } - } - - String url = base + "?action=start&def=reindex&out=text¶ms=fromKrameriusModel," + pid + "," + pid + "&token=" + System.getProperty(ProcessStarter.TOKEN_KEY); - - LOGGER.info("indexer URL:" + url); - try { - ProcessUtils.httpGet(url); - } catch (Exception e) { - LOGGER.severe("Error spawning indexer for " + pid + ":" + e); - } - - } - }, - /** - * Request to add pid to virtual collection - */ - ADDTOCOLLECTION { - - @Override - void doPerform(VirtualCollectionServlet vc, FedoraAccess fedoraAccess, CollectionsManager colMan, HttpServletRequest req, HttpServletResponse resp) throws IOException, SecurityException { - String pid = req.getParameter("pid"); - String collection = req.getParameter("collection"); - CollectionUtils.addToCollection(pid, collection, fedoraAccess); - } - }, - /** - * Request to remove pid from virtual collection - */ - REMOVEFROMCOLLECTION { - - @Override - void doPerform(VirtualCollectionServlet vc, FedoraAccess fedoraAccess, CollectionsManager colMan, HttpServletRequest req, HttpServletResponse resp) throws IOException, SecurityException { - String pid = req.getParameter("pid"); - String collection = req.getParameter("collection"); - CollectionUtils.removeFromCollection(pid, collection, fedoraAccess); - } - }, - /** - * Request to write to response the content of parameter content. - * Used when creating datastream - */ - TEXT { - - @Override - void doPerform(VirtualCollectionServlet vc, FedoraAccess fedoraAccess, CollectionsManager colMan, HttpServletRequest req, HttpServletResponse resp) throws IOException, SecurityException { - String content = req.getParameter("content"); - vc.writeOutput(req, resp, content); - } - }, - /** - * Request to create a virtual collection - */ - LABEL { - - @Override - void doPerform(VirtualCollectionServlet vc, FedoraAccess fedoraAccess, CollectionsManager colMan, HttpServletRequest req, HttpServletResponse resp) throws IOException, SecurityException, CollectionException { - String pid = req.getParameter("pid"); - Collection collection = colMan.getCollection(pid); - if (collection != null) { - vc.writeOutput(req, resp, collection.getLabel()); - } - } - }, - /** - * Request to create a virtual collection - */ - CREATE { - - @Override - void doPerform(VirtualCollectionServlet vc, FedoraAccess fedoraAccess, CollectionsManager colMan, HttpServletRequest req, HttpServletResponse resp) throws IOException, SecurityException, InterruptedException, RepositoryException { - -// boolean canLeave = Boolean.parseBoolean(req.getParameter("canLeave")); -// Map plainTexts = new HashMap(); -// Enumeration paramNames = req.getParameterNames(); -// while(paramNames.hasMoreElements()) { -// String p = paramNames.nextElement().toString(); -// if (p.startsWith("text_")) { -// String langCode = p.substring("text_".length()); -// plainTexts.put(langCode, req.getParameter(p)); -// } -// } -// -// String pid = CollectionUtils.create(fedoraAccess, null, canLeave, plainTexts, new CollectionUtils.CollectionManagerWait(colMan)); -// -// resp.setContentType("text/plain"); -// vc.writeOutput(req, resp, pid); - } - }, - /** - * Request to delete a virtual collection - */ - DELETE { - - @Override - void doPerform(VirtualCollectionServlet vc, FedoraAccess fedoraAccess, CollectionsManager colMan, HttpServletRequest req, HttpServletResponse resp) throws Exception, SecurityException { -// String pid = req.getParameter("pid"); -// CollectionUtils.delete(pid, fedoraAccess); - } - }, - /** - * Request to change a virtual collection - */ - CHANGE { - - @Override - void doPerform(VirtualCollectionServlet vc, FedoraAccess fedoraAccess, CollectionsManager colMan, HttpServletRequest req, HttpServletResponse resp) throws IOException, SecurityException, RepositoryException { -// String[] langs = vc.getLangs(); -// String pid = req.getParameter("pid"); -// boolean canLeave = Boolean.parseBoolean(req.getParameter("canLeave")); -// CollectionUtils.modify(pid, pid, canLeave, fedoraAccess); -// String string = req.getRequestURL().toString(); -// URL url = new URL(string); -// //String k4url = url.getProtocol() + "://" + url.getHost() + ApplicationURL.extractPort(url) + req.getRequestURI(); -// for (int i = 0; i < langs.length; i++) { -// String lang = langs[++i]; -// String text = req.getParameter("text_" + lang); -// if (text != null) { -// CollectionUtils.modifyLangDatastream(pid, lang, text, fedoraAccess); -// } -// } -// PrintWriter out = resp.getWriter(); -// out.print("1"); - - } - }; - - abstract void doPerform(VirtualCollectionServlet vc, FedoraAccess fedoraAccess, CollectionsManager colMan, HttpServletRequest req, HttpServletResponse response) throws Exception, SecurityException; - } -} diff --git a/search/src/java/cz/incad/Kramerius/views/virtualcollection/VirtualCollectionViewObject.java b/search/src/java/cz/incad/Kramerius/views/virtualcollection/VirtualCollectionViewObject.java deleted file mode 100644 index 6b01ca2b09..0000000000 --- a/search/src/java/cz/incad/Kramerius/views/virtualcollection/VirtualCollectionViewObject.java +++ /dev/null @@ -1,241 +0,0 @@ -/* - * Copyright (C) 2011 Alberto Hernandez - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.Kramerius.views.virtualcollection; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; -import java.util.logging.Level; -import java.util.logging.Logger; - -import javax.servlet.http.HttpServletRequest; - -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.google.inject.name.Named; - -import cz.incad.Kramerius.security.strenderers.CollectionsWrapper; -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.security.User; -import cz.incad.kramerius.users.LoggedUsersSingleton; -import cz.incad.kramerius.utils.conf.KConfiguration; -import cz.incad.kramerius.virtualcollections.Collection; -import cz.incad.kramerius.virtualcollections.Collection.Description; -import cz.incad.kramerius.virtualcollections.CollectionsManager; -import cz.incad.kramerius.virtualcollections.CollectionsManager.SortOrder; -import cz.incad.kramerius.virtualcollections.CollectionsManager.SortType; - -public class VirtualCollectionViewObject { - - public static Logger LOGGER = Logger.getLogger(VirtualCollectionViewObject.class.getName()); - - @Inject - Provider requestProvider; - - @Inject - Provider localeProvider; - - - @Inject - Provider virtualCollectionProvider; - - @Inject - KConfiguration kConfiguration; - - @Inject - @Named("securedFedoraAccess") - FedoraAccess fedoraAccess; - - - @Inject - LoggedUsersSingleton loggedUsersSingleton; - - @Inject - Provider userProvider; - - @Inject - @Named("fedora") - CollectionsManager fedoraManager; - - @Inject - @Named("fedora") - CollectionsManager solrManager; - - public List getVirtualCollections() throws Exception { - return wrap(this.solrManager.getCollections()); - } - - - private List wrap(List collections) throws IOException { - List retvals = new ArrayList(); - for (Collection collection : collections) { - retvals.add(new CollectionItemViewObject(collection, this.fedoraAccess)); - } - return retvals; - } - - public List getVirtualCollectionsFromFedora() throws Exception { - List collections = this.fedoraManager.getCollections(); - return wrap(collections); - } - - public List getVirtualCollectionsLocale() throws Exception { - SortOrder selectedVal = sortOrder(); - SortType sortType = sortType(); - if (selectedVal != null) { - if (sortType == null) { - sortType = SortType.ALPHABET; - } - return onlyLocalizedDescriptions(this.solrManager.getSortedCollections(this.localeProvider.get(), selectedVal, sortType)); - } else { - return onlyLocalizedDescriptions(this.solrManager.getCollections()); - } - } - - private SortOrder sortOrder() { - String confString = KConfiguration.getInstance().getConfiguration().getString("search.collection.sort"); - if (confString == null) return null; - SortOrder selectedVal = null; - for (SortOrder v : CollectionsManager.SortOrder.values()) { - if (confString.equals(v.name())) { - selectedVal = v; - break; - } - } - return selectedVal; - } - - private SortType sortType() { - String confString = KConfiguration.getInstance().getConfiguration().getString("search.collection.sortType"); - if (confString == null) return null; - SortType selectedVal = null; - for (SortType v : CollectionsManager.SortType.values()) { - if (confString.equals(v.name())) { - selectedVal = v; - break; - } - } - return selectedVal; - } - - public boolean isThumbnailsVisible() { - boolean thumbs = KConfiguration.getInstance().getConfiguration().getBoolean("search.collection.thumbs",false); - return thumbs; - } - - - private List onlyLocalizedDescriptions(List rawCollection) throws IOException { - Locale locale = this.localeProvider.get(); - List ncols = new ArrayList(); - for (Collection rCol : rawCollection) { - Collection col = new Collection(rCol.getPid(),rCol.getLabel(), rCol.getUrl() ,rCol.isCanLeaveFlag()); - col.setNumberOfDocs(rCol.getNumberOfDocs()); - Description l = rCol.lookup(locale.getLanguage()); - if (l != null) { - col.addDescription(l);; - } - ncols.add(col); - } - return wrap(ncols); - } - - public List getVirtualCollectionsFromFedoraLocale() throws Exception { - SortOrder selectedVal = sortOrder(); - SortType sortType = sortType(); - if (selectedVal != null) { - if (sortType == null) { - sortType = SortType.ALPHABET; - } - return onlyLocalizedDescriptions(this.fedoraManager.getSortedCollections(this.localeProvider.get(), selectedVal, sortType)); - } else { - return onlyLocalizedDescriptions(this.fedoraManager.getCollections()); - } - } - - public Collection getCurrent(){ - return this.virtualCollectionProvider.get(); - - } - - public boolean getCanLeaveCurrent(){ - return this.getCurrent().isCanLeaveFlag(); - } - - public String getCurrentText(){ - Collection c = this.getCurrent(); - Description lookup = c.lookup(this.localeProvider.get().getLanguage()); - return lookup != null ? lookup.getText() : ""; - } - - - public List getHomeTabs() throws Exception{ - String[] tabs = kConfiguration.getPropertyList("search.home.tabs"); - if (!this.loggedUsersSingleton.isLoggedUser(this.requestProvider)) { - tabs= filterLogged(tabs); - } - // Mozne hodnoty custom,mostDesirables,newest,facets,browseAuthor,browseTitle,info - // Pokud mame nastavenou sbirku NEzobrazime mostDesirables, custom, browseAuthor,browseTitle - ArrayList validTabs = new ArrayList(); - Collection vc = getCurrent(); - for(String tab:tabs){ - if(vc == null || - tab.equals("info") || - tab.equals("facets") || - tab.equals("newest") || - (tab.equals("collections") && vc==null && getVirtualCollections().size()>0 ) ){ - validTabs.add(tab); - } - } - - return validTabs; - } - - private String[] filterLogged(String[] tabs) { - List mustBeLoggedList = new ArrayList(Arrays.asList(kConfiguration.getPropertyList("search.home.tabs.onlylogged"))); - List alist = new ArrayList(); - for (int i = 0; i < tabs.length; i++) { - if (!mustBeLoggedList.contains(tabs[i])) { - alist.add(tabs[i]); - } - } - return (String[]) alist.toArray(new String[alist.size()]); - } - - - - public CollectionItemViewObject getParameterCollection() { - try { - HttpServletRequest request = this.requestProvider.get(); - String parameter = request.getParameter("collection"); - if (parameter != null) { - return new CollectionItemViewObject(this.fedoraManager.getCollection(parameter), this.fedoraAccess); - } else return null; - } catch (Exception e) { - LOGGER.log(Level.SEVERE, e.getMessage(),e); - throw new RuntimeException(e); - } - } - - - public String getLocaleLang() { - return this.localeProvider.get().getLanguage(); - } - - -} diff --git a/search/src/test/cz/incad/kramerius/exts/menu/main/MainMenuTest.java b/search/src/test/cz/incad/kramerius/exts/menu/main/MainMenuTest.java deleted file mode 100644 index 251ce7f2cd..0000000000 --- a/search/src/test/cz/incad/kramerius/exts/menu/main/MainMenuTest.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright (C) 2012 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -/** - * - */ -package cz.incad.kramerius.exts.menu.main; - -import java.io.File; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Locale; - -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpSession; - -import junit.framework.Assert; - -import org.easymock.EasyMock; -import org.junit.Test; - -import com.google.inject.AbstractModule; -import com.google.inject.Binding; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Provides; -import com.google.inject.Scopes; -import com.google.inject.name.Named; - -import cz.incad.Kramerius.exts.menu.main.guice.MainMenuConfiguration; -import cz.incad.Kramerius.exts.menu.main.impl.adm.items.IndexerAdministration; -import cz.incad.Kramerius.exts.menu.main.impl.adm.items.NDKMetsImport; -import cz.incad.Kramerius.exts.menu.main.impl.adm.items.ParametrizedConvert; -import cz.incad.kramerius.AbstractGuiceTestCase; -import cz.incad.kramerius.processes.DefinitionManager; -import cz.incad.kramerius.processes.LRProcessManager; -import cz.incad.kramerius.processes.impl.LRProcessDefinitionManagerImpl; -import cz.incad.kramerius.security.SecuredActions; -import cz.incad.kramerius.service.ResourceBundleService; - - - -/** - * @author pavels - * - */ -public class MainMenuTest extends AbstractGuiceTestCase { - - @Test - public void shouldPass() { -// Injector inj = injector(); -// IndexerAdministration reindexDialogItem = inj.getInstance(IndexerAdministration.class); -// Assert.assertTrue(reindexDialogItem.isRenderable()); -// -// ParametrizedConvert parametrizedConvert = inj.getInstance(ParametrizedConvert.class); -// Assert.assertTrue(parametrizedConvert.isRenderable()); -// -// NDKMetsImport metsImport = inj.getInstance(NDKMetsImport.class); -// Assert.assertFalse(metsImport.isRenderable()); - } - - @Override - protected Injector injector() { - HttpServletRequest mockReq = EasyMock.createMock(HttpServletRequest.class); - HttpSession mockSess = EasyMock.createMock(HttpSession.class); - - LRProcessManager processMan = EasyMock.createMock(LRProcessManager.class); - ResourceBundleService resbundleserv = EasyMock.createMock(ResourceBundleService.class); - - EasyMock.expect(mockSess.getAttribute("securityForRepository")).andReturn(Arrays.asList(SecuredActions.REINDEX.getFormalName(), SecuredActions.CONVERT.getFormalName())).anyTimes(); - EasyMock.expect(mockReq.getSession()).andReturn(mockSess).anyTimes(); - EasyMock.replay(mockReq, processMan, resbundleserv, mockSess); - - - Injector injector = Guice.createInjector( - new MainMenuConfiguration(), - new _Module(mockReq, processMan, resbundleserv) - ); - return injector; - } - - public static class _Module extends AbstractModule { - - HttpServletRequest request; - LRProcessManager processManager; - ResourceBundleService resBundleServ; - - public _Module(HttpServletRequest request, LRProcessManager manager, ResourceBundleService resbundle) { - super(); - this.request = request; - this.processManager = manager; - this.resBundleServ = resbundle; - } - - @Override - protected void configure() { - bind(DefinitionManager.class).to(LRProcessDefinitionManagerImpl.class).in(Scopes.SINGLETON); - bind(LRProcessManager.class).toInstance(processManager); - bind(ResourceBundleService.class).toInstance(resBundleServ); - } - - @Provides - public HttpServletRequest getRequest() { - return this.request; - } - - @Provides - public Locale getLocale() { - return Locale.getDefault(); - } - - @Provides - @Named("LIBS") - public String getLibsFolder() { - return "LIBS"; - } - - } -} diff --git a/search/src/test/cz/incad/kramerius/views/PDFGenerateViewObjectTest.java b/search/src/test/cz/incad/kramerius/views/PDFGenerateViewObjectTest.java deleted file mode 100644 index 9d73275ad9..0000000000 --- a/search/src/test/cz/incad/kramerius/views/PDFGenerateViewObjectTest.java +++ /dev/null @@ -1,263 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.kramerius.views; - -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.Charset; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Locale; -import java.util.Map; -import java.util.PropertyResourceBundle; - -import javax.servlet.http.HttpServletRequest; -import javax.xml.parsers.ParserConfigurationException; - -import junit.framework.Assert; - -import org.easymock.EasyMock; -import org.junit.Ignore; -import org.junit.Test; -import org.xml.sax.SAXException; - -import antlr.RecognitionException; -import antlr.TokenStreamException; - -import com.google.inject.AbstractModule; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Provides; -import com.google.inject.name.Names; - -import cz.incad.Kramerius.views.AbstractPrintViewObject.RadioItem; -import cz.incad.Kramerius.views.PdfGenerateViewObject; -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.ObjectPidsPath; -import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.service.ResourceBundleService; -import cz.incad.kramerius.service.TextsService; -import cz.incad.kramerius.utils.XMLUtils; - -@Ignore -// TODO: Rewrite; new index -public class PDFGenerateViewObjectTest { - - public static Map IMG_FULL_MAPPINGS = new HashMap(); static { - IMG_FULL_MAPPINGS.put("uuid:00dbc770-8138-11e0-b63f-000d606f5dc6", true); - IMG_FULL_MAPPINGS.put("uuid:045b1250-7e47-11e0-add1-000d606f5dc6", false); - IMG_FULL_MAPPINGS.put("uuid:17b01dc0-96f7-11de-9176-000d606f5dc6", true); - IMG_FULL_MAPPINGS.put("uuid:8f526130-8b0d-11de-8994-000d606f5dc6", false); - IMG_FULL_MAPPINGS.put("uuid:91214030-80bb-11e0-b482-000d606f5dc6", false); - IMG_FULL_MAPPINGS.put("uuid:ab7e5a19-bddb-11e0-bff9-0016e6840575", false); - IMG_FULL_MAPPINGS.put("uuid:f7e50720-80b6-11e0-9ec7-000d606f5dc6", false); - } - - - public static Map PATHS_MAPPING = new HashMap(); static { - // monograph -> page - PATHS_MAPPING.put("uuid:00dbc770-8138-11e0-b63f-000d606f5dc6", new ObjectPidsPath("uuid:8f526130-8b0d-11de-8994-000d606f5dc6","uuid:00dbc770-8138-11e0-b63f-000d606f5dc6")); - // monograph - PATHS_MAPPING.put("uuid:8f526130-8b0d-11de-8994-000d606f5dc6", new ObjectPidsPath("uuid:8f526130-8b0d-11de-8994-000d606f5dc6")); - // periodikum - PATHS_MAPPING.put("uuid:045b1250-7e47-11e0-add1-000d606f5dc6", new ObjectPidsPath("uuid:045b1250-7e47-11e0-add1-000d606f5dc6")); - - // rocnik - PATHS_MAPPING.put("uuid:f7e50720-80b6-11e0-9ec7-000d606f5dc6", new ObjectPidsPath("uuid:045b1250-7e47-11e0-add1-000d606f5dc6","uuid:f7e50720-80b6-11e0-9ec7-000d606f5dc6")); - - // vytisk - PATHS_MAPPING.put("uuid:91214030-80bb-11e0-b482-000d606f5dc6", new ObjectPidsPath("uuid:045b1250-7e47-11e0-add1-000d606f5dc6","uuid:f7e50720-80b6-11e0-9ec7-000d606f5dc6", "uuid:91214030-80bb-11e0-b482-000d606f5dc6")); - - // stranka - PATHS_MAPPING.put("uuid:17b01dc0-96f7-11de-9176-000d606f5dc6", new ObjectPidsPath("uuid:045b1250-7e47-11e0-add1-000d606f5dc6","uuid:f7e50720-80b6-11e0-9ec7-000d606f5dc6", "uuid:91214030-80bb-11e0-b482-000d606f5dc6","uuid:17b01dc0-96f7-11de-9176-000d606f5dc6")); - - // clanek - PATHS_MAPPING.put("uuid:ab7e5a19-bddb-11e0-bff9-0016e6840575", new ObjectPidsPath("uuid:045b1250-7e47-11e0-add1-000d606f5dc6","uuid:f7e50720-80b6-11e0-9ec7-000d606f5dc6", "uuid:91214030-80bb-11e0-b482-000d606f5dc6","uuid:ab7e5a19-bddb-11e0-bff9-0016e6840575")); - - } - - private static final String TEXT = - ""+ - " Podmínky využití"+ - " "+ - " Knihovna poskytuje přístup k digitalizovaným dokumentům pouze pro nekomerční, vědecké, studijní"+ - " účely a pouze pro osobní potřeby uživatelů. Část dokumentů digitální knihovny podléhá autorským"+ - " právům. Využitím digitální knihovny a vygenerováním kopie části digitalizovaného"+ - " dokumentu se uživatel zavazuje dodržovat tyto podmínky využití, které musí být součástí každé"+ - " zhotovené kopie. Jakékoli další kopírování materiálu z digitální knihovny není možné bez"+ - " případného písemného svolení knihovny."+ - " "+ - ""; - - - @Test - public void testTitles() throws IOException, RecognitionException, TokenStreamException, ParserConfigurationException, SAXException { - Injector injector = prepare(new String[] { - "uuid:8f526130-8b0d-11de-8994-000d606f5dc6", - "uuid:045b1250-7e47-11e0-add1-000d606f5dc6", - "uuid:ab7e5a19-bddb-11e0-bff9-0016e6840575" - }); - - PdfGenerateViewObject view = new PdfGenerateViewObject(); - injector.injectMembers(view); - view.init(); - - Assert.assertEquals(" Podmínky využití", view.getHeader()); - List items = view.getItems(); - Assert.assertTrue(items.size() == 3); - - RadioItem monograph = items.get(0); - Assert.assertTrue(monograph.isMaster()); - Assert.assertEquals("Monografie:Verspätete Erwägungen", monograph.getName()); - Assert.assertTrue(monograph.getPids().size() == 1); - - RadioItem periodical = items.get(1); - Assert.assertTrue(periodical.isMaster()); - Assert.assertEquals("Periodikum:Dějiny a současnost", periodical.getName()); - Assert.assertTrue(periodical.getPids().size() == 1); - - RadioItem intpart = items.get(2); - Assert.assertTrue(intpart.isMaster()); - Assert.assertEquals("Interní součást:„Lidé si budou vypravovat o jejich moudrosti\" | Výtisk:3 | Ročník:33 | Periodikum:Dějiny a současnost", intpart.getName()); - Assert.assertTrue(intpart.getPids().size() == 1); - } - - - - - @Test - public void testTwoPageTitles() throws IOException, RecognitionException, TokenStreamException, ParserConfigurationException, SAXException { - Injector injector = prepare(new String[] { - "uuid:17b01dc0-96f7-11de-9176-000d606f5dc6", - "uuid:00dbc770-8138-11e0-b63f-000d606f5dc6" - }); - - PdfGenerateViewObject view = new PdfGenerateViewObject(); - injector.injectMembers(view); - view.init(); - - List items = view.getItems(); - Assert.assertTrue(items.size() == 1); - RadioItem item = items.get(0); - Assert.assertTrue(item.getPids().size() == 2); - Assert.assertEquals("Stránky: 4 - 6", item.getName()); - - String[] details = item.getDetailedItemNames(); - Assert.assertTrue(details.length == 2); - Assert.assertEquals("Stránka:4 | Výtisk:3 | Ročník:33 | Periodikum:Dějiny a současnost", details[0].trim()); - Assert.assertEquals("Stránka:6 | Monografie:Verspätete Erwägungen", details[1]); - - } - - public Injector prepare(String[] processedPids) throws IOException, ParserConfigurationException, SAXException { - - Locale locale = Locale.getDefault(); - - TextsService textsService = EasyMock.createMock(TextsService.class); - EasyMock.expect(textsService.getText("first_page_nolines_xml", locale)).andReturn( TEXT); - - - FedoraAccess fedoraAccess = EasyMock.createMock(FedoraAccess.class); - List processedPidsAsList = Arrays.asList(processedPids); - for (String pid : processedPids) { - EasyMock.expect(fedoraAccess.isImageFULLAvailable(pid)).andReturn(IMG_FULL_MAPPINGS.get(pid)).anyTimes(); - - InputStream is = PDFGenerateViewObjectTest.class.getResourceAsStream("dc."+pid.replace(":", "")+".xml"); - EasyMock.expect(fedoraAccess.getDC(pid)).andReturn(XMLUtils.parseDocument(is,true)).anyTimes(); - - // dc pro cestu - String[] path = PATHS_MAPPING.get(pid).getPathFromLeafToRoot(); - for (String subPid : path) { - if (!processedPidsAsList.contains(subPid)) { - - InputStream subIs = PDFGenerateViewObjectTest.class.getResourceAsStream("dc."+subPid.replace(":", "")+".xml"); - EasyMock.expect(fedoraAccess.getDC(subPid)).andReturn(XMLUtils.parseDocument(subIs,true)).anyTimes(); - - EasyMock.expect(fedoraAccess.isImageFULLAvailable(subPid)).andReturn(IMG_FULL_MAPPINGS.get(subPid)).anyTimes(); - } - } - } - - - ResourceBundleService bundleService = EasyMock.createMock(ResourceBundleService.class); - InputStream resStream = this.getClass().getClassLoader().getResourceAsStream("labels_cs.properties"); - EasyMock.expect(bundleService.getResourceBundle("labels", locale)).andReturn(new PropertyResourceBundle(new InputStreamReader(resStream, Charset.forName("UTF-8")))).anyTimes(); - - SolrAccess solrAccess = EasyMock.createMock(SolrAccess.class); - for (String pid : processedPids) { - EasyMock.expect(solrAccess.getPidPaths(pid)).andReturn(new ObjectPidsPath[] { PATHS_MAPPING.get(pid) }).anyTimes(); - } - - - HttpServletRequest request = EasyMock.createMock(HttpServletRequest.class); - StringBuilder builder = new StringBuilder(); - for (int i = 0; i < processedPids.length; i++) { - if (i>0) builder.append(";"); - String escapedPid = processedPids[i].replaceAll(":", "\\\\:"); - builder.append(escapedPid); - } - EasyMock.expect(request.getParameter("pids")).andReturn("{"+builder.toString()+"}").anyTimes(); - - - EasyMock.replay(textsService, fedoraAccess, bundleService, solrAccess,request); - - - Injector injector = Guice.createInjector( - new _Module(locale, textsService, fedoraAccess, bundleService, solrAccess, request) - ); - return injector; - } - - - public static class _Module extends AbstractModule { - - private Locale locale; - private TextsService textsService; - private FedoraAccess fa; - private ResourceBundleService bundleService; - private SolrAccess solrAccess; - private HttpServletRequest request; - - - public _Module(Locale locale, TextsService textsService, FedoraAccess fa, ResourceBundleService bundleService, SolrAccess solrAccess, HttpServletRequest req) { - super(); - this.locale = locale; - this.textsService = textsService; - this.fa = fa; - this.bundleService = bundleService; - this.solrAccess = solrAccess; - this.request = req; - } - - - @Override - protected void configure() { - bind(Locale.class).toInstance(this.locale); - bind(TextsService.class).toInstance(this.textsService); - bind(FedoraAccess.class).annotatedWith(Names.named("securedFedoraAccess")).toInstance(this.fa); - bind(SolrAccess.class).toInstance(solrAccess); - bind(ResourceBundleService.class).toInstance(this.bundleService); - } - - @Provides - protected HttpServletRequest getHttpServletRequest() { - return this.request; - } - } -} diff --git a/search/src/test/cz/incad/kramerius/views/ProcessViewObjectTest.java b/search/src/test/cz/incad/kramerius/views/ProcessViewObjectTest.java deleted file mode 100644 index d7d35f72cf..0000000000 --- a/search/src/test/cz/incad/kramerius/views/ProcessViewObjectTest.java +++ /dev/null @@ -1,65 +0,0 @@ -package cz.incad.kramerius.views; - -import java.io.IOException; -import java.io.StringReader; -import java.text.ParseException; -import java.util.PropertyResourceBundle; -import java.util.ResourceBundle; - -import org.easymock.EasyMock; -import org.junit.Assert; -import org.junit.Test; - -import cz.incad.Kramerius.views.ProcessViewObject; -import cz.incad.kramerius.FedoraAccess; - -public class ProcessViewObjectTest { - - public static final String BUNDLES="administrator.processes.duration.days=days \n" + - "administrator.processes.duration.hours=hours\n" + - "administrator.processes.duration.minutes=minutes\n " + - "administrator.processes.duration.seconds=seconds\n" + - "administrator.processes.duration.miliseconds=miliseconds"; - - - @Test - public void testDateCalculations() throws ParseException, IOException { - PropertyResourceBundle propRes = new PropertyResourceBundle(new StringReader(BUNDLES)); - - String start = "2013/03/04 - 08:13:57"; - String stop = "2013/03/04 - 10:08:20"; - StringBuilder calculated = ProcessViewObject.formatDuration(ProcessViewObject.FORMAT.parse(start).getTime(), ProcessViewObject.FORMAT.parse(stop).getTime(), propRes); - Assert.assertTrue(calculated.toString().trim().equals("1 hours 54 minutes 23 seconds")); - - start = "2013/03/04 - 08:13:57"; - stop = "2013/03/04 - 08:13:58"; - calculated = ProcessViewObject.formatDuration(ProcessViewObject.FORMAT.parse(start).getTime(), ProcessViewObject.FORMAT.parse(stop).getTime(), propRes); - Assert.assertTrue(calculated.toString().trim().equals("1 seconds")); - - start = "2013/03/04 - 08:13:57"; - stop = "2013/03/04 - 08:14:58"; - calculated = ProcessViewObject.formatDuration(ProcessViewObject.FORMAT.parse(start).getTime(), ProcessViewObject.FORMAT.parse(stop).getTime(), propRes); - Assert.assertTrue(calculated.toString().trim().equals("1 minutes 1 seconds")); - - start = "2013/03/04 - 08:13:57"; - stop = "2013/03/04 - 08:14:56"; - calculated = ProcessViewObject.formatDuration(ProcessViewObject.FORMAT.parse(start).getTime(), ProcessViewObject.FORMAT.parse(stop).getTime(), propRes); - Assert.assertTrue(calculated.toString().trim().equals("59 seconds")); - - start = "2013/03/04 - 08:13:57"; - stop = "2013/03/04 - 09:14:56"; - calculated = ProcessViewObject.formatDuration(ProcessViewObject.FORMAT.parse(start).getTime(), ProcessViewObject.FORMAT.parse(stop).getTime(), propRes); - Assert.assertTrue(calculated.toString().trim().equals("1 hours 59 seconds")); - - start = "2013/03/04 - 08:13:57"; - stop = "2013/03/04 - 09:16:22"; - calculated = ProcessViewObject.formatDuration(ProcessViewObject.FORMAT.parse(start).getTime(), ProcessViewObject.FORMAT.parse(stop).getTime(), propRes); - Assert.assertTrue(calculated.toString().trim().equals("1 hours 2 minutes 25 seconds")); - - start = "2013/03/04 - 08:13:57"; - stop = "2013/12/04 - 10:33:22"; - calculated = ProcessViewObject.formatDuration(ProcessViewObject.FORMAT.parse(start).getTime(), ProcessViewObject.FORMAT.parse(stop).getTime(), propRes); - Assert.assertTrue(calculated.toString().trim().equals("9 days 2 hours 19 minutes 25 seconds")); - } - -} diff --git a/search/src/test/cz/incad/kramerius/views/ProcessesViewObjectTest.java b/search/src/test/cz/incad/kramerius/views/ProcessesViewObjectTest.java deleted file mode 100644 index cdafe42a7e..0000000000 --- a/search/src/test/cz/incad/kramerius/views/ProcessesViewObjectTest.java +++ /dev/null @@ -1,106 +0,0 @@ -package cz.incad.kramerius.views; - -import javax.servlet.http.HttpServletRequest; - -import junit.framework.Assert; - -import org.easymock.EasyMock; -import org.junit.Test; - -import antlr.RecognitionException; - -import com.google.inject.Provider; - -import cz.incad.Kramerius.views.ProcessesViewObject; -import cz.incad.kramerius.processes.LRProcessManager; -import cz.incad.kramerius.service.TextsService; - -public class ProcessesViewObjectTest { - - - - - private static HttpServletRequest req(String size, String pageNumber, String ordering, String filter) { - //http://krameriusdemo.mzk.cz/search/inc/admin/_processes_data.jsp?size=20&ordering=PLANNED&type=DESC&_=1396603167260 -> last page - HttpServletRequest req = EasyMock.createMock(HttpServletRequest.class); - EasyMock.expect(req.getParameter("size")).andReturn(size).anyTimes(); - EasyMock.expect(req.getParameter("page")).andReturn(pageNumber).anyTimes(); - EasyMock.expect(req.getParameter("ordering")).andReturn(ordering).anyTimes(); - EasyMock.expect(req.getParameter("filter")).andReturn(filter).anyTimes(); - EasyMock.expect(req.getParameter("type")).andReturn(null).anyTimes(); - return req; - } - - @Test - public void testNumberOfPages_OnePage1() throws RecognitionException { - LRProcessManager lrp = EasyMock.createMock(LRProcessManager.class); - HttpServletRequest req = req("20",null, null,null); - - EasyMock.expect(lrp.getNumberOfLongRunningProcesses(null)).andReturn(19); - - - EasyMock.replay(lrp, req); - - ProcessesViewObject pview = new ProcessesViewObject(); - pview.setRequestProvider(new _RequestProvider(req)); - pview.setProcessManager(lrp); - pview.init(); - - int numberOfPages = pview.getNumberOfPages(); - System.out.println(numberOfPages); - System.out.println("firstpage :"+pview.getFirstPage()); - System.out.println("lastpage :"+pview.getLastPage()); - System.out.println("page:"+pview.getPage()); - System.out.println("page next ? :"+pview.getHasNext()); - System.out.println("page previous ? :"+pview.getHasPrevious()); - //Assert.assertTrue(numberOfPages == 1); - - } - - @Test - public void testNumberOfPages_OnePage2() throws RecognitionException { - LRProcessManager lrp = EasyMock.createMock(LRProcessManager.class); - HttpServletRequest req = req("20",null, null,null); - - EasyMock.expect(lrp.getNumberOfLongRunningProcesses(null)).andReturn(27); - - - EasyMock.replay(lrp, req); - - ProcessesViewObject pview = new ProcessesViewObject(); - pview.setRequestProvider(new _RequestProvider(req)); - pview.setProcessManager(lrp); - pview.init(); - - int numberOfPages = pview.getNumberOfPages(); - System.out.println(numberOfPages); - System.out.println("firstpage :"+pview.getFirstPage()); - System.out.println("lastpage :"+pview.getLastPage()); - System.out.println("page:"+pview.getPage()); - System.out.println("page next ? :"+pview.getHasNext()); - System.out.println("page previous ? :"+pview.getHasPrevious()); - - - Assert.assertTrue(numberOfPages == 1); - - } - - private class _RequestProvider implements Provider{ - - private HttpServletRequest req; - - - public _RequestProvider(HttpServletRequest req) { - super(); - this.req = req; - } - - - @Override - public HttpServletRequest get() { - return this.req; - } - - - } -} diff --git a/search/web/WEB-INF/web.xml b/search/web/WEB-INF/web.xml index 0eea4f4449..5323f047f0 100644 --- a/search/web/WEB-INF/web.xml +++ b/search/web/WEB-INF/web.xml @@ -62,7 +62,6 @@ basic /api/* /lr - /stats @@ -92,7 +91,6 @@ - @@ -113,7 +110,6 @@ /* - + diff --git a/security/authfilters/build.gradle b/security/authfilters/build.gradle index 29be2773bc..81d2a5f12d 100644 --- a/security/authfilters/build.gradle +++ b/security/authfilters/build.gradle @@ -7,8 +7,12 @@ dependencies { api "org.brickred:socialauth:4.12" api 'commons-codec:commons-codec:1.5' - api 'org.keycloak:keycloak-servlet-filter-adapter:15.0.2' - api 'org.keycloak:keycloak-authz-client:15.0.2' + //ext.keycloakversion = '20.0.2' + //api 'org.keycloak:keycloak-servlet-filter-adapter:15.0.2' + //api 'org.keycloak:keycloak-authz-client:15.0.2' + + api 'org.keycloak:keycloak-servlet-filter-adapter:20.0.2' + api 'org.keycloak:keycloak-authz-client:20.0.2' } diff --git a/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/impl/AbstractThirdPartyUsersSupport.java b/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/impl/AbstractThirdPartyUsersSupport.java index a60f1bd209..2704097aaf 100644 --- a/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/impl/AbstractThirdPartyUsersSupport.java +++ b/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/impl/AbstractThirdPartyUsersSupport.java @@ -145,6 +145,8 @@ public synchronized String storeUserPropertiesToSession(HttpServletRequest req, } } + + return password; } diff --git a/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/Keycloack3rdUser.java b/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/Keycloack3rdUser.java index a8641d9a18..3ff5a6d265 100644 --- a/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/Keycloack3rdUser.java +++ b/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/Keycloack3rdUser.java @@ -8,9 +8,13 @@ import cz.incad.kramerius.security.impl.UserImpl; import java.util.*; +import java.util.logging.Logger; +import java.util.stream.Collectors; public class Keycloack3rdUser extends AbstractThirdPartyUser { + public static final Logger LOGGER = Logger.getLogger(Keycloack3rdUser.class.getName()); + private String hash; private Map sessionAttributes = new HashMap<>(); @@ -53,7 +57,10 @@ public User toUser(UserManager userManager) { if (!associatedRoles.contains(DefaultRoles.COMMON_USERS.getName())) { associatedRoles.add(DefaultRoles.COMMON_USERS.getName()); } - + + LOGGER.fine(String.format("Associted roles %s",associatedRoles.toString())); + + User user = super.toUser(userManager); // TODO: Change it if (user instanceof UserImpl) { @@ -65,6 +72,8 @@ public User toUser(UserManager userManager) { ((UserImpl) user).setGroups(roles); } + LOGGER.fine(String.format("Returning wrapped user %s (%s)",user.getLoginname(), Arrays.stream(user.getGroups()).map(Role::getName).collect(Collectors.joining(",")))); + return user; } diff --git a/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackFilter.java b/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackFilter.java index aa557698fa..ac950a6e08 100644 --- a/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackFilter.java +++ b/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackFilter.java @@ -32,6 +32,9 @@ protected ThirdPartyUsersSupport getThirdPartyUsersSupport() { protected boolean userStoreIsNeeded(HttpServletRequest httpReq) { try { KeycloakAccount keycloakAccount = (KeycloakAccount) httpReq.getAttribute(KeycloakAccount.class.getName()); + if (keycloakAccount != null && keycloakAccount.getRoles() != null) { + LOGGER.log(Level.FINE, String.format("Keycloak principal %s (%s)",keycloakAccount.getPrincipal().getName(), keycloakAccount.getRoles().toString())); + } return keycloakAccount != null ; }catch (Throwable th){ LOGGER.log(Level.INFO,"Error retrieving KeycloakAccount", th); diff --git a/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java b/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java index 876ba31921..2cf895732f 100644 --- a/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java +++ b/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java @@ -24,6 +24,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -133,11 +134,33 @@ protected Keycloack3rdUser createUserWrapper(HttpServletRequest req, String user keycloack3rdUser.setRoles(new ArrayList<>(roleSet)); AccessToken token = ((KeycloakPrincipal) req.getUserPrincipal()).getKeycloakSecurityContext().getToken(); + String hash = token.getAccessTokenHash(); + String codeHash = token.getCodeHash(); + Long auth_time = token.getAuth_time(); + + keycloack3rdUser.setProperty(UserUtils.FIRST_NAME_KEY, token.getGivenName()); keycloack3rdUser.setProperty(UserUtils.LAST_NAME_KEY, token.getFamilyName()); keycloack3rdUser.setProperty(UserUtils.EMAIL_KEY, token.getEmail()); keycloack3rdUser.setProperty("preffered_user_name", token.getPreferredUsername()); + keycloack3rdUser.setProperty("expiration_time", ""+token.getExp()); + keycloack3rdUser.setProperty("authentication_time", ""+token.getAuth_time()); + keycloack3rdUser.setProperty("preffered_user_name", token.getPreferredUsername()); + keycloack3rdUser.setProperty("expires_in", ""+(token.getExp()-token.getAuth_time())); + keycloack3rdUser.setProperty("token_id", ""+token.getId()); + keycloack3rdUser.setProperty("remote_user", ""+token.getPreferredUsername()); + + + LOGGER.fine("Token id: "+token.getId() +", and returned claims:"+ token.getOtherClaims()); + Map otherClaims = token.getOtherClaims(); + otherClaims.keySet().forEach(key-> { + Object object = otherClaims.get(key); + if (object != null) { + LOGGER.log(Level.FINE,"Key value "+key+" = "+object.toString()); + keycloack3rdUser.setProperty(key, object.toString()); + } + }); return keycloack3rdUser; } diff --git a/security/oauth/src/main/java/cz/incad/kramerius/keycloak/KrameriusKeycloakFilter.java b/security/oauth/src/main/java/cz/incad/kramerius/keycloak/KrameriusKeycloakFilter.java index b5b73f6dc4..b20f191511 100644 --- a/security/oauth/src/main/java/cz/incad/kramerius/keycloak/KrameriusKeycloakFilter.java +++ b/security/oauth/src/main/java/cz/incad/kramerius/keycloak/KrameriusKeycloakFilter.java @@ -19,13 +19,15 @@ import java.util.logging.Logger; import java.util.regex.Pattern; +import static cz.incad.kramerius.Constants.WORKING_DIR; + /** * @author Bill Burke * @version $Revision: 1 $ */ public class KrameriusKeycloakFilter implements Filter { - private final static Logger log = Logger.getLogger("" + KrameriusKeycloakFilter.class); + private final static Logger log = Logger.getLogger(KrameriusKeycloakFilter.class.getName()); public static final String SKIP_PATTERN_PARAM = "keycloak.config.skipPattern"; @@ -77,27 +79,26 @@ public void init(final FilterConfig filterConfig) throws ServletException { deploymentContext = new AdapterDeploymentContext(configResolver); log.log(Level.INFO, "Using {0} to resolve Keycloak configuration on a per-request basis.", configResolverClass); } catch (Exception ex) { - log.log(Level.FINE, "The specified resolver {0} could NOT be loaded. Keycloak is unconfigured and will deny all requests. Reason: {1}", new Object[]{configResolverClass, ex.getMessage()}); + log.log(Level.SEVERE, "The specified resolver {0} could NOT be loaded. Keycloak is unconfigured and will deny all requests. Reason: {1}", new Object[]{configResolverClass, ex.getMessage()}); deploymentContext = new AdapterDeploymentContext(new KeycloakDeployment()); } } else { String fp = filterConfig.getInitParameter(CONFIG_FILE_PARAM); InputStream is = null; - if (fp != null) { - try { + try { + if (fp != null) { is = new FileInputStream(fp); - } catch (FileNotFoundException e) { - throw new RuntimeException(e); + } else { + String path = WORKING_DIR + "/keycloak.json"; + is = new FileInputStream(path); } - } else { - String path = "/WEB-INF/keycloak.json"; - String pathParam = filterConfig.getInitParameter(CONFIG_PATH_PARAM); - if (pathParam != null) path = pathParam; - is = filterConfig.getServletContext().getResourceAsStream(path); + KeycloakDeployment kd = createKeycloakDeploymentFrom(is); + deploymentContext = new AdapterDeploymentContext(kd); + log.fine("Keycloak is using a per-deployment configuration."); + } catch (FileNotFoundException e) { + log.log(Level.SEVERE,"Keycloak is unconfigured and will deny all requests." , e); + deploymentContext = new AdapterDeploymentContext(new KeycloakDeployment()); } - KeycloakDeployment kd = createKeycloakDeploymentFrom(is); - deploymentContext = new AdapterDeploymentContext(kd); - log.fine("Keycloak is using a per-deployment configuration."); } } filterConfig.getServletContext().setAttribute(AdapterDeploymentContext.class.getName(), deploymentContext); @@ -171,14 +172,17 @@ public void logoutHttpSessions(List ids) { return; } AuthenticatedActionsHandler actions = new AuthenticatedActionsHandler(deployment, facade); - if (actions.handledRequest()) { - return; - } else { - HttpServletRequestWrapper wrapper = tokenStore.buildWrapper(); - chain.doFilter(wrapper, res); - return; - } +// if (actions.handledRequest()) { +// return; +// } else { + HttpServletRequestWrapper wrapper = tokenStore.buildWrapper(); + chain.doFilter(wrapper, res); + return; +// } + } else { + log.fine("NOT AUTHENTICATED"); } + // AuthChallenge challenge = authenticator.getChallenge(); // if (challenge != null) { // log.fine("challenge"); @@ -186,7 +190,7 @@ public void logoutHttpSessions(List ids) { // return; // } // response.sendError(403); - chain.doFilter(req,res); + chain.doFilter(req, res); } /** @@ -213,5 +217,6 @@ private boolean shouldSkip(HttpServletRequest request) { public void destroy() { } + } diff --git a/shared/common/build.gradle b/shared/common/build.gradle index 0b51949179..74641cbdae 100644 --- a/shared/common/build.gradle +++ b/shared/common/build.gradle @@ -70,7 +70,7 @@ dependencies { api "commons-dbcp:commons-dbcp:1.4" api "net.sf.json-lib:json-lib:2.3:jdk15" - api 'javax.ws.rs:jsr311-api:1.1.1' + //api 'javax.ws.rs:jsr311-api:1.1.1' api "com.sun.jersey:jersey-server:${jerseyversion}" api "com.sun.jersey:jersey-servlet:${jerseyversion}" diff --git a/shared/common/src/main/java/cz/incad/kramerius/audio/AudioStreamForwardUtils.java b/shared/common/src/main/java/cz/incad/kramerius/audio/AudioStreamForwardUtils.java index a1c8cd2970..a588947fc8 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/audio/AudioStreamForwardUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/audio/AudioStreamForwardUtils.java @@ -34,7 +34,7 @@ public class AudioStreamForwardUtils { public static boolean canBeRead(String pid, SolrAccess sa, User user, RightsResolver rightsResolver) throws IOException { ObjectPidsPath[] paths = sa.getPidPaths(pid); for (ObjectPidsPath pth : paths) { - if (rightsResolver.isActionAllowed(user, SecuredActions.READ.getFormalName(), pid, null, pth.injectRepository()).flag()) { + if (rightsResolver.isActionAllowed(user, SecuredActions.A_READ.getFormalName(), pid, null, pth.injectRepository()).flag()) { return true; } } @@ -61,7 +61,7 @@ public static ResponseBuilder GET(AudioStreamId id, HttpServletRequest request, throw new IllegalArgumentException(ex); } } else { - throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.READ, id.getPid())); + throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.A_READ, id.getPid())); } } @@ -84,7 +84,7 @@ public static void GET(AudioStreamId id, HttpServletRequest request, throw new ServletException(ex); } } else { - throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.READ, id.getPid())); + throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.A_READ, id.getPid())); } } @@ -106,7 +106,7 @@ public static void HEAD(AudioStreamId id, HttpServletRequest request, throw new ServletException(ex); } } else { - throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.READ, id.getPid())); + throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.A_READ, id.getPid())); } } @@ -128,7 +128,7 @@ public static ResponseBuilder HEAD(AudioStreamId id, HttpServletRequest request, throw new IllegalArgumentException(ex); } } else { - throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.READ, id.getPid())); + throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.A_READ, id.getPid())); } } diff --git a/shared/common/src/main/java/cz/incad/kramerius/fedora/om/impl/AkubraRepository.java b/shared/common/src/main/java/cz/incad/kramerius/fedora/om/impl/AkubraRepository.java index 5c4f6d9421..0ebc9908fb 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/fedora/om/impl/AkubraRepository.java +++ b/shared/common/src/main/java/cz/incad/kramerius/fedora/om/impl/AkubraRepository.java @@ -50,9 +50,9 @@ private AkubraRepository(ProcessingIndexFeeder feeder, AkubraDOManager manager) this.manager = manager; } + /** * Create new repository object - * * @param feeder Feeder instance * @param manager * @return diff --git a/shared/common/src/main/java/cz/incad/kramerius/repository/RepositoryApi.java b/shared/common/src/main/java/cz/incad/kramerius/repository/RepositoryApi.java index cecf4d2f92..204e016c10 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/repository/RepositoryApi.java +++ b/shared/common/src/main/java/cz/incad/kramerius/repository/RepositoryApi.java @@ -65,6 +65,14 @@ public interface RepositoryApi { public List getPidsOfObjectsByModel(String model) throws RepositoryException, IOException, SolrServerException; + public Pair> getPidsOfObjectsByModel(String model, int rows, int pageIndex) throws RepositoryException, IOException, SolrServerException; + + + public Pair> getPidsOfObjectsByModel(String model, String titlePrefix, int rows, int pageIndex) throws RepositoryException, IOException, SolrServerException; + + + + public TitlePidPairs getPidsOfObjectsWithTitlesByModel(String model, boolean ascendingOrder, int offset, int limit) throws RepositoryException, IOException, SolrServerException; public TitlePidPairs getPidsOfObjectsWithTitlesByModelWithCursor(String model, boolean ascendingOrder, String cursor, int limit) throws RepositoryException, IOException, SolrServerException; diff --git a/shared/common/src/main/java/cz/incad/kramerius/repository/RepositoryApiImpl.java b/shared/common/src/main/java/cz/incad/kramerius/repository/RepositoryApiImpl.java index dcb26a4604..30bb2601ce 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/repository/RepositoryApiImpl.java +++ b/shared/common/src/main/java/cz/incad/kramerius/repository/RepositoryApiImpl.java @@ -12,9 +12,11 @@ import cz.incad.kramerius.repository.utils.Utils; import cz.incad.kramerius.resourceindex.ProcessingIndexFeeder; import cz.incad.kramerius.utils.Dom4jUtils; +import cz.incad.kramerius.utils.StringUtils; import cz.incad.kramerius.utils.conf.KConfiguration; import cz.incad.kramerius.utils.java.Pair; import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.SolrDocument; import org.dom4j.*; import org.ehcache.CacheManager; @@ -27,9 +29,11 @@ import java.time.LocalDateTime; import java.time.format.DateTimeParseException; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; public class RepositoryApiImpl implements RepositoryApi { @@ -176,6 +180,35 @@ public List getPidsOfAllObjects() throws RepositoryException, IOExceptio return pids; } + @Override + public Pair> getPidsOfObjectsByModel(String model, int rows, int pageIndex) throws RepositoryException, IOException, SolrServerException { + String query = String.format("type:description AND model:%s", "model\\:" + model); //prvni "model:" je filtr na solr pole, druhy "model:" je hodnota pole, coze uprime zbytecne + org.apache.commons.lang3.tuple.Pair> cp = akubraRepository.getProcessingIndexFeeder().getPageSortedByTitle(query, rows, pageIndex, Arrays.asList("source")); + Long numberOfRecords = cp.getLeft(); + List pids = cp.getRight().stream().map(sd-> { + Object fieldValue = sd.getFieldValue("source"); + return fieldValue.toString(); + }).collect(Collectors.toList()); + return new Pair<>(numberOfRecords, pids); + } + + //TODO : Should be replaced by pairs + @Override + public Pair> getPidsOfObjectsByModel(String model, String titlePrefix, int rows, int pageIndex) throws RepositoryException, IOException, SolrServerException { + String query = String.format("type:description AND model:%s", "model\\:" + model); + if (StringUtils.isAnyString(titlePrefix)) { + query = String.format("type:description AND model:%s AND title_edge:%s", "model\\:" + model, titlePrefix); //prvni "model:" je filtr na solr pole, druhy "model:" je hodnota pole, coze uprime zbytecne + } + org.apache.commons.lang3.tuple.Pair> cp = akubraRepository.getProcessingIndexFeeder().getPageSortedByTitle(query, rows, pageIndex, Arrays.asList("source")); + Long numberOfRecords = cp.getLeft(); + List pids = cp.getRight().stream().map(sd-> { + Object fieldValue = sd.getFieldValue("source"); + return fieldValue.toString(); + }).collect(Collectors.toList()); + return new Pair<>(numberOfRecords, pids); + } + + @Override public List getPidsOfObjectsByModel(String model) throws RepositoryException, IOException, SolrServerException { List pids = new ArrayList<>(); diff --git a/shared/common/src/main/java/cz/incad/kramerius/resourceindex/ProcessingIndexFeeder.java b/shared/common/src/main/java/cz/incad/kramerius/resourceindex/ProcessingIndexFeeder.java index 40d686d43c..1383e6c9de 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/resourceindex/ProcessingIndexFeeder.java +++ b/shared/common/src/main/java/cz/incad/kramerius/resourceindex/ProcessingIndexFeeder.java @@ -135,6 +135,29 @@ public void iterateProcessingSortedByIndexationDate(String query, boolean ascend public void iterateProcessingSortedByTitle(String query, Consumer action) throws IOException, SolrServerException { iterateProcessingWithSort(query, "dc.title", SolrQuery.ORDER.asc, action); } + public Pair> getPageSortedByTitle(String query, int rows, int pageIndex, List fieldList) throws IOException, SolrServerException { + List docs = new ArrayList<>(); + SolrQuery solrQuery = new SolrQuery(query); + + int offset = pageIndex*rows; + solrQuery.setStart(offset).setRows(rows); + solrQuery.setSort("title", SolrQuery.ORDER.asc); + + if (fieldList != null && !fieldList.isEmpty()) { + String[] fl = fieldList.toArray(new String[fieldList.size()]); + solrQuery.setFields(fl); + } + QueryResponse response = this.solrClient.query(solrQuery); + long numFound = response.getResults().getNumFound(); + response.getResults().forEach((doc) -> { + docs.add(doc); + }); + return Pair.of(numFound, docs); + } + + public Pair> getPageSortedByTitle(String query, int rows, int offset) throws IOException, SolrServerException { + return getPageSortedByTitle(query, rows, offset, new ArrayList<>()); + } private void iterateProcessingWithSort(String query, String sortField, SolrQuery.ORDER order, Consumer action) throws IOException, SolrServerException { ///String query = "*:*"; diff --git a/shared/common/src/main/java/cz/incad/kramerius/resourceindex/ProcessingIndexRebuildFromFoxmlByPid.java b/shared/common/src/main/java/cz/incad/kramerius/resourceindex/ProcessingIndexRebuildFromFoxmlByPid.java index 6b95aae63e..134e7b3421 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/resourceindex/ProcessingIndexRebuildFromFoxmlByPid.java +++ b/shared/common/src/main/java/cz/incad/kramerius/resourceindex/ProcessingIndexRebuildFromFoxmlByPid.java @@ -47,7 +47,7 @@ private ProcessingIndexRebuildFromFoxmlByPid() { * args[0] - authToken * args[1] - pid */ - public static void main(String[] args) throws IOException, SolrServerException, RepositoryException, FcrepoOperationFailedException { + public static void main(String[] args) throws IOException, SolrServerException, RepositoryException { //args /*LOGGER.info("args: " + Arrays.asList(args)); for (String arg : args) { @@ -62,7 +62,7 @@ public static void main(String[] args) throws IOException, SolrServerException, //process params String pid = args[argsIndex++]; - ProcessStarter.updateName(String.format("Aktualizace Processing indexu z FOXML objektu %s", pid)); + ProcessStarter.updateName(String.format("Aktualizace Processing indexu z FOXML pro objekt %s", pid)); new ProcessingIndexRebuildFromFoxmlByPid().rebuildProcessingIndexFromFoxml(pid); } diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/SecuredActions.java b/shared/common/src/main/java/cz/incad/kramerius/security/SecuredActions.java index fbb256a2bb..614847652d 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/SecuredActions.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/SecuredActions.java @@ -17,123 +17,116 @@ package cz.incad.kramerius.security; /** - * Vyctovy typ chranenych akci systemu K4 + * Vyctovy typ chranenych akci systemu K7 + * */ public enum SecuredActions { - /** Akce cist */ - READ("read"), + /** Definice novych akci**/ + /** READ; ma pravo cist konkretni objekt */ + A_READ("a_read", false), - /** Akce pro proces import */ - IMPORT( "import"), + /** READ; ma pravo vyuzit pdf resource */ + A_PDF_READ("a_pdf_read", false), - /** Akce pro proces konvert */ - CONVERT( "convert"), + /** smazani zaznamu */ + A_DELETE("a_delete",false), - /** Akce pro proces replicationrights */ - REPLICATIONRIGHTS("replicationrights"), + /** spravovat vsechny procesy; smazat, prohlizet logy */ + A_PROCESS_EDIT("a_process_edit"), - /** Akce pro proces enumerator */ - ENUMERATOR( "enumerator"), - - /** Akce pro proces indexace */ - REINDEX( "reindex"), + /** Pravo pro cteni procesu */ + A_PROCESS_READ("a_process_read"), - /** Akce pro proces replikace - periodika */ - REPLIKATOR_PERIODICALS( "replikator_periodicals"), - /** Akce pro proces replikace - monografie */ - REPLIKATOR_MONOGRAPHS( "replikator_monographs"), + /** Spravovat pouze vlastni procesy */ + A_OWNER_PROCESS_EDIT("a_owner_process_edit"), + + /** spusteni indexeru; cely repozitar*/ + A_INDEX("a_index", false), - REPLIKATOR_K3("replikator_k3"), + /** indexace ve stronu */ + //A_INDEX_CHILDREN("a_index_children"), + + /** process rebuild indexu */ + A_REBUILD_PROCESSING_INDEX("a_rebuild_processing_index", false), - /** Akce pro proces delete */ - DELETE( "delete"), + /** import */ + A_IMPORT("a_import"), + + /** pravo nastavovat priznak viditlnosti a licence */ + A_SET_ACCESSIBILITY("a_set_accessibility", false), + - /** Akce pro proces export FOXML*/ - EXPORT( "export"), + /** export pro cdk */ + A_EXPORT_CDK("a_export_cdk"), - /** Akce pro nastave priznaku PRIVATE, PUBLIC */ - SETPRIVATE( "setprivate"), SETPUBLIC( "setpublic"), - - ADMINISTRATE( "administrate"), - - /** Akce pro spusteni metadata editoru */ - EDITOR("editor"), + /** zobrazeni statistik */ + A_STATISTICS("a_statistics"), - /** Akce umozni spravu procesu */ - MANAGE_LR_PROCESS("manage_lr_process"), + /** moznost mazat statistiky */ + A_STATISTICS_EDIT("a_statistics_edit"), - /** Akce pro exportovani replikacnich souboru (deskriptor, foxml, atd..) */ - EXPORT_K4_REPLICATIONS("export_k4_replications"), + /** replikace - export */ + A_EXPORT_REPLICATIONS("a_export_replications"), - IMPORT_K4_REPLICATIONS("import_k4_replications"), + /** replikace - import */ + A_IMPORT_REPLICATIONS("a_import_replications"), + + /** editace prav, pro vsechny objekty krome sbirek*/ + A_RIGHTS_EDIT("a_rights_edit",false), + + /** Pravo cist criteria */ + A_CRITERIA_READ("a_criteria_read"), + /** Cteni kolekci, pravo umoznujici cist informace z admin ponitu pro ceti */ + A_COLLECTIONS_READ("a_collections_read"), - /** Akce pro poskytovani dat pro CDK */ - EXPORT_CDK_REPLICATIONS("export_cdk_replications"), + /** editace kolekci, pridavani do kolekci atd.. */ + A_COLLECTIONS_EDIT("a_collections_edit", false), - //K4_REPLICATIONS("k4_replications"), + /** pravo byti zaraditelny do kolekce */ + A_ABLE_TOBE_PART_OF_COLLECTION("a_able_tobe_part_of_collections", false), - // Issue 159 - @Deprecated - EDIT_INFO_TEXT("edit_info_text"), - /** Akce umozni spoustet editor uzivatelu superadmin modu */ - USERSADMIN("rightsadmin"), + /** spusteni nkp logu */ + A_GENERATE_NKPLOGS("a_generate_nkplogs"), - /** Akce umoznu spoustet editor v subadmin modu */ - USERSSUBADMIN("rightssubadmin"), - - /** Akce umoznuje spravu virtualnich sbirek */ - VIRTUALCOLLECTION_MANAGE("virtualcollection_manage"), + /** editace roli */ + A_ROLES_EDIT("a_roles_edit"), - /** Akce umoznuje sprav u kriterii */ - CRITERIA_RIGHTS_MANAGE("criteria_rights_manage"), - - /** Akce NDK Mets import */ - NDK_METS_IMPORT("ndk_mets_import"), - - /** aggregate process */ - AGGREGATE_PROCESSES("aggregate"), - - /** sorting */ - SORT("sort"), - - - /** page info */ - SHOW_ALTERNATIVE_INFO_TEXT("show_alternative_info_text"), - - /** Akce umozni zobrazit administratorskou cast menu */ - DISPLAY_ADMIN_MENU("display_admin_menu"), + A_ROLES_READ("a_roles_read"), - /** show statistics item */ - SHOW_STATISTICS("show_statictics"), - - /** show print admin menu item */ - SHOW_PRINT_MENU("show_print_menu"), - - SHOW_CLIENT_PRINT_MENU("show_client_print_menu"), - - /** show pdf menu item */ - SHOW_CLIENT_PDF_MENU("show_client_pdf_menu"), + /* Will be enabled in the future + A_USERS_EDIT("a_users_edit"), + A_USERS_READ("a_users_read"), + */ + + /** Pravo pro admin cteni admina */ + A_ADMIN_READ("a_admin_read"); + - /** pdf resource */ - PDF_RESOURCE("pdf_resource"), - - /** DNNT administrate */ - DNNT_ADMIN("dnnt_admin"); - private String formalName; - + private boolean onlyGlobalAction; + + private SecuredActions(String formalName, boolean gA) { + this.formalName = formalName; + this.onlyGlobalAction = gA; + } + private SecuredActions(String formalName) { this.formalName = formalName; + this.onlyGlobalAction = true; } - + public String getFormalName() { return formalName; } + public boolean isGlobalAction() { + return onlyGlobalAction; + } + public static SecuredActions findByFormalName(String fname) { SecuredActions[] vals = values(); @@ -144,4 +137,5 @@ public static SecuredActions findByFormalName(String fname) { } return null; } + } diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/SecuredFedoraAccessImpl.java b/shared/common/src/main/java/cz/incad/kramerius/security/SecuredFedoraAccessImpl.java index da423aa180..75f2a03539 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/SecuredFedoraAccessImpl.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/SecuredFedoraAccessImpl.java @@ -86,11 +86,11 @@ public InputStream getImageFULL(String pid) throws IOException { ObjectPidsPath[] paths = this.solrAccess.getPidPaths(pid); paths = ensurePidPathForUnindexedObjects(pid, paths); for (ObjectPidsPath path : paths) { - if (this.rightsResolver.isActionAllowed(SecuredActions.READ.getFormalName(), pid, FedoraUtils.IMG_FULL_STREAM, path).flag()) { + if (this.rightsResolver.isActionAllowed(SecuredActions.A_READ.getFormalName(), pid, FedoraUtils.IMG_FULL_STREAM, path).flag()) { return rawAccess.getImageFULL(pid); } } - throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.READ, pid, FedoraUtils.IMG_FULL_STREAM)); + throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.A_READ, pid, FedoraUtils.IMG_FULL_STREAM)); } @Override @@ -184,7 +184,7 @@ public boolean isContentAccessible(String pid) throws IOException { ObjectPidsPath[] paths = this.solrAccess.getPidPaths(pid); paths = ensurePidPathForUnindexedObjects(pid, paths); for (ObjectPidsPath path : paths) { - if (this.rightsResolver.isActionAllowed(SecuredActions.READ.getFormalName(), pid, FedoraUtils.IMG_FULL_STREAM, path).flag()) { + if (this.rightsResolver.isActionAllowed(SecuredActions.A_READ.getFormalName(), pid, FedoraUtils.IMG_FULL_STREAM, path).flag()) { return true; } } @@ -225,11 +225,11 @@ public InputStream getFoxml(String pid, boolean archive) throws IOException { ObjectPidsPath[] paths = this.solrAccess.getPidPaths(pid); paths = ensurePidPathForUnindexedObjects(pid, paths); for (int i = 0; i < paths.length; i++) { - if (this.rightsResolver.isActionAllowed(SecuredActions.READ.getFormalName(), pid, null, paths[i]).flag()) { + if (this.rightsResolver.isActionAllowed(SecuredActions.A_READ.getFormalName(), pid, null, paths[i]).flag()) { return rawAccess.getFoxml(pid, archive); } } - throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.READ, pid, null)); + throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.A_READ, pid, null)); } private ObjectPidsPath[] ensurePidPathForUnindexedObjects(String pid, ObjectPidsPath[] paths) throws IOException { @@ -249,12 +249,12 @@ public InputStream getDataStream(String pid, String datastreamName) throws IOExc ObjectPidsPath[] paths = this.solrAccess.getPidPaths(pid); paths = ensurePidPathForUnindexedObjects(pid, paths); for (int i = 0; i < paths.length; i++) { - if (this.rightsResolver.isActionAllowed(SecuredActions.READ.getFormalName(), pid, datastreamName, paths[i]).flag()) { + if (this.rightsResolver.isActionAllowed(SecuredActions.A_READ.getFormalName(), pid, datastreamName, paths[i]).flag()) { return rawAccess.getDataStream(pid, datastreamName); } } - throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.READ, pid, datastreamName)); + throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.A_READ, pid, datastreamName)); } else { String[] securedStreamsExtension = KConfiguration.getInstance().getSecuredAditionalStreams(); int indexOf = Arrays.asList(securedStreamsExtension).indexOf(datastreamName); @@ -262,11 +262,11 @@ public InputStream getDataStream(String pid, String datastreamName) throws IOExc ObjectPidsPath[] paths = this.solrAccess.getPidPaths(pid + "/" + datastreamName); paths = ensurePidPathForUnindexedObjects(pid, paths); for (int i = 0; i < paths.length; i++) { - if (this.rightsResolver.isActionAllowed(SecuredActions.READ.getFormalName(), pid, datastreamName, paths[i]).flag()) { + if (this.rightsResolver.isActionAllowed(SecuredActions.A_READ.getFormalName(), pid, datastreamName, paths[i]).flag()) { return rawAccess.getDataStream(pid, datastreamName); } } - throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.READ, pid, datastreamName)); + throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.A_READ, pid, datastreamName)); } else { return rawAccess.getDataStream(pid, datastreamName); } @@ -310,7 +310,7 @@ public InputStream getFullThumbnail(String pid) throws IOException { ObjectPidsPath[] paths = this.solrAccess.getPidPaths(pid); paths = ensurePidPathForUnindexedObjects(pid, paths); for (ObjectPidsPath path : paths) { - if (this.rightsResolver.isActionAllowed(SecuredActions.READ.getFormalName(), pid, FedoraUtils.IMG_PREVIEW_STREAM, path).flag()) { + if (this.rightsResolver.isActionAllowed(SecuredActions.A_READ.getFormalName(), pid, FedoraUtils.IMG_PREVIEW_STREAM, path).flag()) { accessed = true; break; } @@ -323,7 +323,7 @@ public InputStream getFullThumbnail(String pid) throws IOException { throw new IOException("preview not found"); } } else { - throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.READ, pid, FedoraUtils.IMG_PREVIEW_STREAM)); + throw new SecurityException(new SecurityException.SecurityExceptionInfo(SecuredActions.A_READ, pid, FedoraUtils.IMG_PREVIEW_STREAM)); } } diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/Abonents.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/Abonents.java index c84e72e691..d768b91ada 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/Abonents.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/Abonents.java @@ -64,7 +64,7 @@ public boolean isParamsNecessary() { @Override public SecuredActions[] getApplicableActions() { - return new SecuredActions[] {SecuredActions.READ}; + return new SecuredActions[] {SecuredActions.A_READ}; } @Override diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/BenevolentModelFilter.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/BenevolentModelFilter.java index 93ce99fa9b..876005f2cb 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/BenevolentModelFilter.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/BenevolentModelFilter.java @@ -86,9 +86,7 @@ static boolean containsModelName(Object[] objects, String modelName) { @Override public SecuredActions[] getApplicableActions() { return new SecuredActions[] { - SecuredActions.READ, - SecuredActions.SHOW_CLIENT_PRINT_MENU, - SecuredActions.SHOW_CLIENT_PDF_MENU + SecuredActions.A_READ }; } @@ -107,5 +105,4 @@ public RightCriteriumPriorityHint getPriorityHint() { public boolean isParamsNecessary() { return true; } - } diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/BenevolentMovingWall.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/BenevolentMovingWall.java index 6cf03fe319..1b6f287c16 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/BenevolentMovingWall.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/BenevolentMovingWall.java @@ -128,7 +128,7 @@ public boolean isParamsNecessary() { @Override public SecuredActions[] getApplicableActions() { - return new SecuredActions[] { SecuredActions.READ }; + return new SecuredActions[] { SecuredActions.A_READ }; } @Override diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/CoverAndContentFilter.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/CoverAndContentFilter.java index b4fb300658..7265167ae4 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/CoverAndContentFilter.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/CoverAndContentFilter.java @@ -2,8 +2,11 @@ import cz.incad.kramerius.FedoraAccess; import cz.incad.kramerius.FedoraNamespaceContext; +import cz.incad.kramerius.SolrAccess; import cz.incad.kramerius.security.*; import cz.incad.kramerius.utils.XMLUtils; +import cz.incad.kramerius.utils.solr.SolrUtils; + import org.w3c.dom.Document; import org.xml.sax.SAXException; @@ -45,12 +48,14 @@ public EvaluatingResultState evalute() throws RightCriteriumException { if ("page".equals(fedoraAccess.getKrameriusModelName(pid))) { Document mods = XMLUtils.parseDocument( fedoraAccess.getDataStream(pid, "BIBLIO_MODS"), true); + if (checkTypeElement(mods).equals(EvaluatingResultState.TRUE)) + return isNotPeriodical(pid); return checkTypeElement(mods); } else { return EvaluatingResultState.NOT_APPLICABLE; } } else { - return EvaluatingResultState.TRUE; + return EvaluatingResultState.NOT_APPLICABLE; } } catch (IOException | SAXException | ParserConfigurationException e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); @@ -73,6 +78,24 @@ private EvaluatingResultState checkTypeElement(Document mods) throws IOException } } + private EvaluatingResultState isNotPeriodical(String pid) throws IOException { + try { + //SolrAccess solrAccess = getEvaluateContext().getSolrAccess(); + SolrAccess solrAccess = getEvaluateContext().getSolrAccessNewIndex(); + Document doc = solrAccess.getSolrDataByPid(pid); //SolrUtils.getSolrDataInternal(SolrUtils.UUID_QUERY + "\"" + pid + "\""); + String rootPID = SolrUtils.disectRootPid(doc); + doc = solrAccess.getSolrDataByPid(rootPID); + String rootFedoraModel = SolrUtils.disectFedoraModel(doc); + if (rootFedoraModel.equals("periodical")) + return EvaluatingResultState.NOT_APPLICABLE; + else + return EvaluatingResultState.TRUE; + } catch (XPathExpressionException ex) { + LOGGER.log(Level.SEVERE, null, ex); + } + return EvaluatingResultState.TRUE; + } + private void initModsTypeExpr() throws IOException { try { XPath xpath = XPathFactory.newInstance().newXPath(); @@ -104,6 +127,6 @@ public boolean isParamsNecessary() { @Override public SecuredActions[] getApplicableActions() { - return new SecuredActions[]{SecuredActions.READ}; + return new SecuredActions[]{SecuredActions.A_READ}; } } diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/MovingWall.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/MovingWall.java index bba4b41c86..b8db2bf3be 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/MovingWall.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/MovingWall.java @@ -63,8 +63,7 @@ * (konkretni monografii, konkretni periodikum, atd..) */ public class MovingWall extends AbstractCriterium implements RightCriterium { - - //encoding="marc" + //encoding="marc" public static String[] MODS_XPATHS = { "//mods:originInfo/mods:dateIssued[@encoding='marc']/text()", "//mods:originInfo/mods:dateIssued/text()", @@ -233,7 +232,7 @@ public EvaluatingResultState mockEvaluate(DataMockExpectation dataMockExpectatio } private Document solrDocument(String pid) throws IOException, ParserConfigurationException, SAXException { - SolrAccess solrAccess = this.getEvaluateContext().getSolrAccess(); + SolrAccess solrAccess = this.getEvaluateContext().getSolrAccessNewIndex(); return solrAccess.getSolrDataByPid(pid); //return SolrUtils.getSolrDataInternal(SolrUtils.UUID_QUERY + "\"" + pid + "\""); } @@ -489,7 +488,7 @@ public boolean isParamsNecessary() { @Override public SecuredActions[] getApplicableActions() { - return new SecuredActions[]{SecuredActions.READ}; + return new SecuredActions[]{SecuredActions.A_READ}; } @Override diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/NegativeBenevolentModelFilter.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/NegativeBenevolentModelFilter.java index 5a9a446be4..c99f7c405c 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/NegativeBenevolentModelFilter.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/NegativeBenevolentModelFilter.java @@ -48,9 +48,7 @@ public EvaluatingResultState mockEvaluate(DataMockExpectation dataMockExpectatio @Override public SecuredActions[] getApplicableActions() { return new SecuredActions[] { - SecuredActions.READ, - SecuredActions.SHOW_CLIENT_PRINT_MENU, - SecuredActions.SHOW_CLIENT_PDF_MENU + SecuredActions.A_READ }; } diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/PDFDNNTFlag.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/PDFDNNTFlag.java deleted file mode 100644 index a366394db7..0000000000 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/PDFDNNTFlag.java +++ /dev/null @@ -1,76 +0,0 @@ -package cz.incad.kramerius.security.impl.criteria; - -import cz.incad.kramerius.ObjectPidsPath; -import cz.incad.kramerius.security.*; -import cz.incad.kramerius.security.impl.criteria.utils.CriteriaDNNTUtils; -import cz.incad.kramerius.utils.solr.SolrUtils; - -import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; - -import static cz.incad.kramerius.security.impl.criteria.utils.CriteriaDNNTUtils.checkContainsCriterium; - -// PDF Flag is not used; remove -@Deprecated -public class PDFDNNTFlag extends AbstractCriterium { - - public transient static final Logger LOGGER = Logger.getLogger(ReadDNNTFlag.class.getName()); - - - @Override - public EvaluatingResultState evalute() throws RightCriteriumException { - String requestedPid = null; - try { - requestedPid = this.getEvaluateContext().getRequestedPid(); - if (requestedPid != null && !SpecialObjects.isSpecialObject(requestedPid)) { - // only if - String s = SolrUtils.disectDNNTFlag(this.evalContext.getSolrAccess().getSolrDataByPid(requestedPid).getDocumentElement()); - if (s != null && s.equals("true")) { - RightsResolver rightsResolver = this.getEvaluateContext().getRightsResolver(); - ObjectPidsPath[] paths = this.getEvaluateContext().getSolrAccess().getPidPaths(requestedPid); - for (ObjectPidsPath path : paths) { - RightsReturnObject obj = rightsResolver.isActionAllowed(SecuredActions.READ.getFormalName(), requestedPid, null, path); - if (CriteriaDNNTUtils.allowedByReadDNNTFlagRight(obj)) return EvaluatingResultState.FALSE; - } - } - } - } catch (IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } - - // not applicable - return EvaluatingResultState.NOT_APPLICABLE; - } - - - @Override - public EvaluatingResultState mockEvaluate(DataMockExpectation dataMockExpectation) throws RightCriteriumException { - return EvaluatingResultState.NOT_APPLICABLE; - } - - @Override - public RightCriteriumPriorityHint getPriorityHint() { - return RightCriteriumPriorityHint.DNNT_EXCLUSIVE_MAX; - } - - @Override - public boolean isParamsNecessary() { - return false; - } - - @Override - public SecuredActions[] getApplicableActions() { - return new SecuredActions[] {SecuredActions.PDF_RESOURCE}; - } - - @Override - public boolean isRootLevelCriterum() { - return true; - } - - @Override - public void checkPrecodition(RightsManager manager) throws CriteriaPrecoditionException { - //checkContainsCriterium(this.evalContext, manager); - } -} diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/PDFDNNTLabels.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/PDFDNNTLabels.java index 13d3726595..5a157341a9 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/PDFDNNTLabels.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/PDFDNNTLabels.java @@ -26,7 +26,7 @@ public EvaluatingResultState evalute() throws RightCriteriumException { RightsResolver rightsResolver = this.getEvaluateContext().getRightsResolver(); ObjectPidsPath[] paths = this.getEvaluateContext().getSolrAccessNewIndex().getPidPaths(requestedPid); for (ObjectPidsPath path : paths) { - RightsReturnObject obj = rightsResolver.isActionAllowed(SecuredActions.READ.getFormalName(), requestedPid, null, path); + RightsReturnObject obj = rightsResolver.isActionAllowed(SecuredActions.A_READ.getFormalName(), requestedPid, null, path); if (CriteriaDNNTUtils.allowedByReadDNNTLabelsRight(obj, getLicense())) return EvaluatingResultState.FALSE; } } @@ -65,7 +65,7 @@ public boolean isParamsNecessary() { @Override public SecuredActions[] getApplicableActions() { - return new SecuredActions[] {SecuredActions.PDF_RESOURCE}; + return new SecuredActions[] {SecuredActions.A_PDF_READ}; } @Override diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/PolicyFlag.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/PolicyFlag.java index b2a7b6a1be..5b5dc8a005 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/PolicyFlag.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/PolicyFlag.java @@ -61,7 +61,7 @@ public boolean isParamsNecessary() { @Override public SecuredActions[] getApplicableActions() { - return new SecuredActions[] {SecuredActions.READ}; + return new SecuredActions[] {SecuredActions.A_READ}; } diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/ReadDNNTFlag.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/ReadDNNTFlag.java deleted file mode 100644 index 28ad68889d..0000000000 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/ReadDNNTFlag.java +++ /dev/null @@ -1,74 +0,0 @@ -package cz.incad.kramerius.security.impl.criteria; - - -import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.security.*; -import cz.incad.kramerius.utils.solr.SolrUtils; -import org.w3c.dom.Document; - -import java.io.IOException; -import java.util.logging.Level; -import java.util.logging.Logger; - -// DNNT flag is not used; remove -@Deprecated -public class ReadDNNTFlag extends AbstractCriterium { - - public transient static final Logger LOGGER = Logger.getLogger(ReadDNNTFlag.class.getName()); - - @Override - public EvaluatingResultState evalute() throws RightCriteriumException { - try { - RightCriteriumContext ctx = getEvaluateContext(); - String pid = ctx.getRequestedPid(); - if (!SpecialObjects.isSpecialObject(pid)) { - if (!pid.equals(SpecialObjects.REPOSITORY.getPid())) { - SolrAccess solrAccess = ctx.getSolrAccess(); - Document doc = solrAccess.getSolrDataByPid(pid); - String val = SolrUtils.disectDNNTFlag(doc.getDocumentElement()); - return (val != null && val.equals("true")) ? EvaluatingResultState.TRUE : EvaluatingResultState.NOT_APPLICABLE; - - } else return EvaluatingResultState.NOT_APPLICABLE; - - } else return EvaluatingResultState.NOT_APPLICABLE; - } catch (IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - return EvaluatingResultState.NOT_APPLICABLE; - } - } - - @Override - public EvaluatingResultState mockEvaluate(DataMockExpectation dataMockExpectation) throws RightCriteriumException { - switch (dataMockExpectation) { - case EXPECT_DATA_VAUE_EXISTS: return EvaluatingResultState.TRUE; - case EXPECT_DATA_VALUE_DOESNTEXIST: return EvaluatingResultState.NOT_APPLICABLE; - } - return EvaluatingResultState.NOT_APPLICABLE; - } - - @Override - public RightCriteriumPriorityHint getPriorityHint() { - return RightCriteriumPriorityHint.DNNT_EXCLUSIVE_MIN; - } - - @Override - public boolean isParamsNecessary() { - return false; - } - - @Override - public SecuredActions[] getApplicableActions() { - return new SecuredActions[] {SecuredActions.READ}; - } - - @Override - public boolean isRootLevelCriterum() { - return false; - } - - @Override - public void checkPrecodition(RightsManager manager) throws CriteriaPrecoditionException { - //checkContainsCriteriumPDFDNNT(this.evalContext, manager); - } - -} \ No newline at end of file diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/ReadDNNTFlagIPFiltered.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/ReadDNNTFlagIPFiltered.java deleted file mode 100644 index 32f446fb90..0000000000 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/ReadDNNTFlagIPFiltered.java +++ /dev/null @@ -1,64 +0,0 @@ -package cz.incad.kramerius.security.impl.criteria; - -import static cz.incad.kramerius.security.impl.criteria.utils.CriteriaDNNTUtils.*; - -import cz.incad.kramerius.security.*; - -import java.util.logging.Logger; - -import static cz.incad.kramerius.security.impl.criteria.utils.CriteriaIPAddrUtils.matchIPAddresses; - -// TODO: DNNT Flag is not used; remove -@Deprecated -public class ReadDNNTFlagIPFiltered extends AbstractCriterium { - - public transient static final Logger LOGGER = Logger.getLogger(ReadDNNTFlagIPFiltered.class.getName()); - - @Override - public RightCriteriumPriorityHint getPriorityHint() { - return RightCriteriumPriorityHint.DNNT_EXCLUSIVE_MIN; - } - - @Override - public boolean isParamsNecessary() { - return true; - } - - @Override - public EvaluatingResultState evalute() throws RightCriteriumException { - String pid = getEvaluateContext().getRequestedPid(); - if (!SpecialObjects.isSpecialObject(pid)) { - EvaluatingResultState state = checkDnnt(getEvaluateContext()); - if (!pid.equals(SpecialObjects.REPOSITORY.getPid())) { - if (state == EvaluatingResultState.TRUE) { - EvaluatingResultState result = matchIPAddresses(super.getEvaluateContext(), getObjects()) ? EvaluatingResultState.TRUE : EvaluatingResultState.NOT_APPLICABLE; - return result; - } else { - return state; - } - } else return EvaluatingResultState.NOT_APPLICABLE; - - } else return EvaluatingResultState.NOT_APPLICABLE; - } - - @Override - public EvaluatingResultState mockEvaluate(DataMockExpectation dataMockExpectation) throws RightCriteriumException { - return matchIPAddresses(super.getEvaluateContext(), getObjects()) ? EvaluatingResultState.TRUE : EvaluatingResultState.NOT_APPLICABLE; - } - - @Override - public void checkPrecodition(RightsManager manager) throws CriteriaPrecoditionException { - //checkContainsCriteriumPDFDNNT(this.evalContext, manager); - } - - - @Override - public SecuredActions[] getApplicableActions() { - return new SecuredActions[]{SecuredActions.READ}; - } - - @Override - public boolean isRootLevelCriterum() { - return false; - } -} \ No newline at end of file diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/ReadDNNTLabels.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/ReadDNNTLabels.java index 660a3e407c..23f3944ca2 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/ReadDNNTLabels.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/ReadDNNTLabels.java @@ -13,8 +13,11 @@ //TODO: Rename to ReadLicense public class ReadDNNTLabels extends AbstractCriterium implements RightCriteriumLabelAware{ + // backward compatibility public static final String PROVIDED_BY_DNNT_LABEL = "providedByLabel"; + public static final String PROVIDED_BY_DNNT_LICENSE = "providedByLicense"; + public transient static final Logger LOGGER = Logger.getLogger(ReadDNNTLabels.class.getName()); private License license; @@ -35,6 +38,7 @@ public EvaluatingResultState evalute() throws RightCriteriumException { if (applied) { // select label getEvaluateContext().getEvaluateInfoMap().put(ReadDNNTLabels.PROVIDED_BY_DNNT_LABEL, getLicense().getName()); + getEvaluateContext().getEvaluateInfoMap().put(ReadDNNTLabels.PROVIDED_BY_DNNT_LICENSE, getLicense().getName()); return EvaluatingResultState.TRUE; } } @@ -67,7 +71,7 @@ public boolean isParamsNecessary() { @Override public SecuredActions[] getApplicableActions() { - return new SecuredActions[] {SecuredActions.READ}; + return new SecuredActions[] {SecuredActions.A_READ}; } @Override diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/ReadDNNTLabelsIPFiltered.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/ReadDNNTLabelsIPFiltered.java index 912e357202..4e9aaa5091 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/ReadDNNTLabelsIPFiltered.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/ReadDNNTLabelsIPFiltered.java @@ -34,6 +34,7 @@ public EvaluatingResultState evalute() throws RightCriteriumException { EvaluatingResultState result = matchIPAddresses(super.getEvaluateContext(), getObjects()) ? EvaluatingResultState.TRUE : EvaluatingResultState.NOT_APPLICABLE; if (result.equals(EvaluatingResultState.TRUE)) { getEvaluateContext().getEvaluateInfoMap().put(ReadDNNTLabels.PROVIDED_BY_DNNT_LABEL, getLicense().getName()); + getEvaluateContext().getEvaluateInfoMap().put(ReadDNNTLabels.PROVIDED_BY_DNNT_LICENSE, getLicense().getName()); } return result; @@ -64,7 +65,7 @@ public boolean isParamsNecessary() { @Override public SecuredActions[] getApplicableActions() { - return new SecuredActions[] {SecuredActions.READ}; + return new SecuredActions[] {SecuredActions.A_READ}; } @Override diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/SecuredStreams.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/SecuredStreams.java index 4534e907dc..430541309a 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/SecuredStreams.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/SecuredStreams.java @@ -42,8 +42,6 @@ public boolean isParamsNecessary() { @Override public SecuredActions[] getApplicableActions() { - return new SecuredActions[] {SecuredActions.READ}; + return new SecuredActions[] {SecuredActions.A_READ}; } - - } diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/Window.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/Window.java index feadc48aec..66f981de89 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/Window.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/Window.java @@ -166,7 +166,7 @@ public boolean isParamsNecessary() { @Override public SecuredActions[] getApplicableActions() { - return new SecuredActions[] { SecuredActions.READ }; + return new SecuredActions[] { SecuredActions.A_READ }; } @Override diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/utils/CriteriaDNNTUtils.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/utils/CriteriaDNNTUtils.java index a3fb613fb5..7f670057c3 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/utils/CriteriaDNNTUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/utils/CriteriaDNNTUtils.java @@ -26,7 +26,7 @@ public class CriteriaDNNTUtils { // check dnnt flag from solr public static EvaluatingResultState checkDnnt(RightCriteriumContext ctx) { try { - SolrAccess solrAccess = ctx.getSolrAccess(); + SolrAccess solrAccess = ctx.getSolrAccessNewIndex(); String pid = ctx.getRequestedPid(); Document doc = solrAccess.getSolrDataByPid(pid); String val = SolrUtils.disectDNNTFlag(doc.getDocumentElement()); @@ -54,8 +54,9 @@ public static boolean allowedByReadDNNTLabelsRight(RightsReturnObject obj, Licen // allowed by dnnt right public static boolean allowedByReadDNNTFlagRight(RightsReturnObject obj) { if (obj.getRight() != null && obj.getRight().getCriteriumWrapper() != null) { - if ( obj.getRight().getCriteriumWrapper().getRightCriterium().getQName().equals(ReadDNNTFlag.class.getName()) || - obj.getRight().getCriteriumWrapper().getRightCriterium().getQName().equals(ReadDNNTFlagIPFiltered.class.getName()) || + if ( /*obj.getRight().getCriteriumWrapper().getRightCriterium().getQName().equals(ReadDNNTFlag.class.getName()) || + obj.getRight().getCriteriumWrapper().getRightCriterium().getQName().equals(ReadDNNTFlagIPFiltered.class.getName()) ||*/ + obj.getRight().getCriteriumWrapper().getRightCriterium().getQName().equals(ReadDNNTLabels.class.getName()) || obj.getRight().getCriteriumWrapper().getRightCriterium().getQName().equals(ReadDNNTLabelsIPFiltered.class.getName()) ) { @@ -80,15 +81,10 @@ public static boolean allowedByReadDNNTFlagRight(RightsReturnObject obj) { // } - // check if there is - public static void checkContainsCriteriumPDFDNNT(RightCriteriumContext ctx, RightsManager manager) throws CriteriaPrecoditionException { - //PDFDNNTFlag.class.getName() - checkContainsCriterium(ctx, manager, PDFDNNTFlag.class); - } public static void checkContainsCriterium(RightCriteriumContext ctx, RightsManager manager, Class ... clzs) throws CriteriaPrecoditionException { String[] pids = new String[] {SpecialObjects.REPOSITORY.getPid()}; - Right[] rights = manager.findRights(pids, SecuredActions.PDF_RESOURCE.getFormalName(), ctx.getUser()); + Right[] rights = manager.findRights(pids, SecuredActions.A_PDF_READ.getFormalName(), ctx.getUser()); for (Right r : rights) { if (r == null) continue; if (r.getCriteriumWrapper() == null) continue; @@ -104,9 +100,6 @@ public static void checkContainsCriterium(RightCriteriumContext ctx, RightsManag throw new CriteriaPrecoditionException("These flags are not set : "+collections); } - public static void checkContainsCriterium(RightCriteriumContext ctx, RightsManager manager) throws CriteriaPrecoditionException { - checkContainsCriterium(ctx, manager, PDFDNNTFlag.class); - } public static boolean matchLicense(Document solrDoc, License license) { List indexedLabels = SolrUtils.disectLicenses(solrDoc.getDocumentElement()); diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/utils/CriteriaRELSEXTUtils.java b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/utils/CriteriaRELSEXTUtils.java index dc805ebcd0..f25c3369ae 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/utils/CriteriaRELSEXTUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/impl/criteria/utils/CriteriaRELSEXTUtils.java @@ -5,14 +5,16 @@ import cz.incad.kramerius.security.EvaluatingResultState; import cz.incad.kramerius.security.RightCriteriumContext; import cz.incad.kramerius.security.SpecialObjects; -import cz.incad.kramerius.security.impl.criteria.ReadDNNTFlag; import org.w3c.dom.Document; import javax.xml.xpath.*; import java.io.IOException; import java.util.logging.Level; +import java.util.logging.Logger; public class CriteriaRELSEXTUtils { + + public static final Logger LOGGER = Logger.getLogger(CriteriaRELSEXTUtils.class.getName()); protected static Object valueFromRELSEXT(Document relsExt, String path) throws XPathExpressionException { XPathFactory xpfactory = XPathFactory.newInstance(); @@ -43,10 +45,10 @@ public static EvaluatingResultState evaluateState(RightCriteriumContext ctx, Str if (!requestedPID.equals(SpecialObjects.REPOSITORY.getPid())) { Document relsExt = fa.getRelsExt(requestedPID); return checkValue(relsExt, path,expectedValue); - } else return EvaluatingResultState.TRUE; + } else return EvaluatingResultState.NOT_APPLICABLE; } catch (IOException e) { - ReadDNNTFlag.LOGGER.log(Level.SEVERE, e.getMessage(), e); - return EvaluatingResultState.TRUE; + LOGGER.log(Level.SEVERE, e.getMessage(), e); + return EvaluatingResultState.NOT_APPLICABLE; } } } \ No newline at end of file diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/res/criteriums b/shared/common/src/main/java/cz/incad/kramerius/security/res/criteriums index f5f08e66c1..efc07d6a2d 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/res/criteriums +++ b/shared/common/src/main/java/cz/incad/kramerius/security/res/criteriums @@ -10,9 +10,6 @@ cz.incad.kramerius.security.impl.criteria.DefaultDomainFilter cz.incad.kramerius.security.impl.criteria.StrictDomainFilter cz.incad.kramerius.security.impl.criteria.NegativeBenevolentModelFilter cz.incad.kramerius.security.impl.criteria.BenevolentModelFilter -cz.incad.kramerius.security.impl.criteria.ReadDNNTFlagIPFiltered -cz.incad.kramerius.security.impl.criteria.ReadDNNTFlag -cz.incad.kramerius.security.impl.criteria.PDFDNNTFlag cz.incad.kramerius.security.impl.criteria.ReadDNNTLabels cz.incad.kramerius.security.impl.criteria.ReadDNNTLabelsIPFiltered cz.incad.kramerius.security.impl.criteria.PDFDNNTLabels diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/StatisticReport.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/StatisticReport.java index 60e4d3fdd1..94f8d43a23 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/StatisticReport.java +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/StatisticReport.java @@ -32,6 +32,9 @@ public interface StatisticReport { /** Simple date format */ public static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat("yyyy.MM.dd"); + + public static final SimpleDateFormat SOLR_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); + //"2017-11-19 22:52:42.738" public static final SimpleDateFormat TIMESTAMP_FORMAT = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss.SSS"); @@ -42,10 +45,13 @@ public interface StatisticReport { public static final String ACTION_KEY = "action"; public static final String LANG_KEY = "lang"; public static final String AUTHOR_NAME_KEY = "author_name"; - + + public static final String PROVIDED_LICENSE_KEY="provided_by_license"; + public static final String DATE_FROM = "from"; public static final String DATE_TO = "to"; - + + /** @@ -66,7 +72,7 @@ public interface StatisticReport { * * @return */ - public List getOptionalValues(); + public List getOptionalValues(StatisticsFiltersContainer filters); /** * Return report identifier @@ -75,11 +81,7 @@ public interface StatisticReport { */ public String getReportId(); - /** - * Prepares view necessary for rendering plot - * @param action - */ - public void prepareViews(ReportedAction action, StatisticsFiltersContainer container) throws StatisticsReportException ; + public boolean convertToObject(); /** @@ -94,8 +96,5 @@ public interface StatisticReport { * @param action Reporting action * @param container Filters container */ - public boolean verifyFilters(ReportedAction action, StatisticsFiltersContainer container); - - - + public List verifyFilters(ReportedAction action, StatisticsFiltersContainer container); } diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/StatisticsAccessLog.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/StatisticsAccessLog.java index 9002571b79..47875a0269 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/StatisticsAccessLog.java +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/StatisticsAccessLog.java @@ -27,7 +27,6 @@ * @author pavels */ public interface StatisticsAccessLog { - /** * Report one access * @param pid accessing pid @@ -64,6 +63,7 @@ public interface StatisticsAccessLog { * @return */ public StatisticReport[] getAllReports(); + /** * Find report by given id @@ -72,7 +72,20 @@ public interface StatisticsAccessLog { */ public StatisticReport getReportById(String reportId); + + /** + * TODO: Presunout jinam + * Cleaning statistics tables + * @param dateFrom From date + * @param dateTo To date + */ + public int cleanData(Date dateFrom, Date dateTo) throws IOException; + /** + * TODO: Presunout jinam + * @throws IOException + */ + public void refresh() throws IOException; } diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/accesslogs/AggregatedAccessLogs.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/accesslogs/AggregatedAccessLogs.java index 8742059e6d..bcdcd6a365 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/accesslogs/AggregatedAccessLogs.java +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/accesslogs/AggregatedAccessLogs.java @@ -8,6 +8,7 @@ import cz.incad.kramerius.statistics.StatisticsAccessLogSupport; import java.io.IOException; +import java.util.Date; public class AggregatedAccessLogs implements StatisticsAccessLog{ @@ -56,4 +57,14 @@ public StatisticReport[] getAllReports() { public StatisticReport getReportById(String reportId) { return this.databaseAccessLog.getReportById(reportId); } + + @Override + public int cleanData(Date dateFrom, Date dateTo) { + throw new UnsupportedOperationException("unsupported"); + } + + @Override + public void refresh() throws IOException { + } + } diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/accesslogs/database/DatabaseStatisticsAccessLogImpl.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/accesslogs/database/DatabaseStatisticsAccessLogImpl.java index ffe934acf4..cfcf26b7ca 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/accesslogs/database/DatabaseStatisticsAccessLogImpl.java +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/accesslogs/database/DatabaseStatisticsAccessLogImpl.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.charset.Charset; import java.sql.Connection; import java.sql.Date; import java.sql.ResultSet; @@ -42,7 +43,9 @@ import cz.incad.kramerius.security.impl.criteria.utils.CriteriaDNNTUtils; import cz.incad.kramerius.statistics.accesslogs.AbstractStatisticsAccessLog; import cz.incad.kramerius.statistics.accesslogs.utils.SElemUtils; +import cz.incad.kramerius.statistics.database.StatisticDbInitializer; import cz.incad.kramerius.utils.DatabaseUtils; +import cz.incad.kramerius.utils.IOUtils; import cz.incad.kramerius.utils.solr.SolrUtils; import org.antlr.stringtemplate.StringTemplate; import org.antlr.stringtemplate.StringTemplateGroup; @@ -88,7 +91,7 @@ public class DatabaseStatisticsAccessLogImpl extends AbstractStatisticsAccessLog @Inject @Named("new-index") SolrAccess solrAccess; - + @Inject @Named("cachedFedoraAccess") FedoraAccess fedoraAccess; @@ -98,10 +101,10 @@ public class DatabaseStatisticsAccessLogImpl extends AbstractStatisticsAccessLog @Inject LoggedUsersSingleton loggedUsersSingleton; - + @Inject Provider userProvider; - + @Inject Set reports; @@ -140,6 +143,7 @@ public void reportAccess(final String pid, final String streamName) throws IOExc RightsReturnObject rightsReturnObject = CriteriaDNNTUtils.currentThreadReturnObject.get(); Map evaluateInfoMap = rightsReturnObject != null ? rightsReturnObject.getEvaluateInfoMap() : new HashMap<>(); + // insert standardni zaznam commands.add(new InsertRecord(pid, loggedUsersSingleton, requestProvider, userProvider, this.reportedAction.get(), false, false, evaluateInfoMap, user.getSessionAttributes(), versionService.getVersion(), licenses)); for (int i = 0, ll = paths.length; i < ll; i++) { @@ -153,61 +157,66 @@ public void reportAccess(final String pid, final String streamName) throws IOExc final String detailPid = pathFromLeafToRoot[j]; String kModel = fedoraAccess.getKrameriusModelName(detailPid); - Document dc = fedoraAccess.getDC(detailPid); - Document sDoc = this.solrAccess.getSolrDataByPid(pid); + //Document sDoc = this.solrAccess.getSolrDataByPid(pid); + Document dc = null; + try { + dc = fedoraAccess.getDC(detailPid); + } catch (IOException e) { + LOGGER.warning("datastream DC not found for " + detailPid + ", ignoring statistics"); + } + if (dc != null) { + Object dateFromDC = DCUtils.dateFromDC(dc); + dateFromDC = dateFromDC != null ? dateFromDC : new JDBCUpdateTemplate.NullObject(String.class); - Object dateFromDC = DCUtils.dateFromDC(dc); - dateFromDC = dateFromDC != null ? dateFromDC : new JDBCUpdateTemplate.NullObject(String.class); + Object dateFromSolr = SElemUtils.selem("str", "datum_str", solrDoc); + dateFromSolr = dateFromSolr != null ? dateFromSolr : new JDBCUpdateTemplate.NullObject(String.class); - Object dateFromSolr = SElemUtils.selem("str", "datum_str", solrDoc); - dateFromSolr = dateFromSolr != null ? dateFromSolr : new JDBCUpdateTemplate.NullObject(String.class); + Object languageFromDc = DCUtils.languageFromDC(dc); + languageFromDc = languageFromDc != null ? languageFromDc : new JDBCUpdateTemplate.NullObject(String.class); - Object languageFromDc = DCUtils.languageFromDC(dc); - languageFromDc = languageFromDc != null ? languageFromDc : new JDBCUpdateTemplate.NullObject(String.class); - - Object title = DCUtils.titleFromDC(dc); - title = title != null ? title : new JDBCUpdateTemplate.NullObject(String.class); + Object title = DCUtils.titleFromDC(dc); + title = title != null ? title : new JDBCUpdateTemplate.NullObject(String.class); - Object rights = DCUtils.rightsFromDC(dc); - rights = rights != null ? rights : new JDBCUpdateTemplate.NullObject(String.class); + Object rights = DCUtils.rightsFromDC(dc); + rights = rights != null ? rights : new JDBCUpdateTemplate.NullObject(String.class); - Document mods = fedoraAccess.getBiblioMods(detailPid); - List languagesFromMods = null; + Document mods = fedoraAccess.getBiblioMods(detailPid); + List languagesFromMods = null; - Map> identifiers = null; - try { - identifiers = ModsUtils.identifiersFromMods(mods); - } catch (XPathExpressionException e) { - Logger.getLogger(DatabaseStatisticsAccessLogImpl.class.getName()).log(Level.SEVERE, e.getMessage(), e); - } + Map> identifiers = null; + try { + identifiers = ModsUtils.identifiersFromMods(mods); + } catch (XPathExpressionException e) { + Logger.getLogger(DatabaseStatisticsAccessLogImpl.class.getName()).log(Level.SEVERE, e.getMessage(), e); + } - try { - languagesFromMods = ModsUtils.languagesFromMods(mods); - } catch (XPathExpressionException ex) { - Logger.getLogger(DatabaseStatisticsAccessLogImpl.class.getName()).log(Level.SEVERE, ex.getMessage(), ex); - } - - if (!languagesFromMods.isEmpty()) { - for (String languageFromMods : languagesFromMods) { - InsertDetail insertDetail = new InsertDetail(detailPid, kModel, rights, dateFromDC, dateFromSolr, languageFromMods, title, identifiers, pathIndex); + try { + languagesFromMods = ModsUtils.languagesFromMods(mods); + } catch (XPathExpressionException ex) { + Logger.getLogger(DatabaseStatisticsAccessLogImpl.class.getName()).log(Level.SEVERE, ex.getMessage(), ex); + } + + if (!languagesFromMods.isEmpty()) { + for (String languageFromMods : languagesFromMods) { + InsertDetail insertDetail = new InsertDetail(detailPid, kModel, rights, dateFromDC, dateFromSolr, languageFromMods, title, identifiers, pathIndex); + commands.add(insertDetail); + } + } else { + InsertDetail insertDetail = new InsertDetail(detailPid, kModel, rights, dateFromDC, dateFromSolr, languageFromDc, title, identifiers, pathIndex); commands.add(insertDetail); } - } - else { - InsertDetail insertDetail = new InsertDetail(detailPid, kModel, rights, dateFromDC, dateFromSolr, languageFromDc, title, identifiers, pathIndex); - commands.add(insertDetail); - } - String[] creatorsFromDC = DCUtils.creatorsFromDC(dc); - for (String cr : creatorsFromDC) { - InsertAuthor insertAuth = new InsertAuthor(cr); - commands.add(insertAuth); - } + String[] creatorsFromDC = DCUtils.creatorsFromDC(dc); + for (String cr : creatorsFromDC) { + InsertAuthor insertAuth = new InsertAuthor(cr); + commands.add(insertAuth); + } - String[] publishersFromDC = DCUtils.publishersFromDC(dc); - for (String p : publishersFromDC) { - InsertPublisher inserPublisher = new InsertPublisher(p); - commands.add(inserPublisher); + String[] publishersFromDC = DCUtils.publishersFromDC(dc); + for (String p : publishersFromDC) { + InsertPublisher inserPublisher = new InsertPublisher(p); + commands.add(inserPublisher); + } } } } @@ -224,8 +233,6 @@ public void reportAccess(final String pid, final String streamName) throws IOExc } - - @Override public void reportAccess(String pid, String streamName, String actionName) throws IOException { ReportedAction action = ReportedAction.valueOf(actionName); @@ -234,12 +241,10 @@ public void reportAccess(String pid, String streamName, String actionName) throw } - @Override public boolean isReportingAccess(String pid, String streamName) { - return streamName.equals(ImageStreams.IMG_FULL.name()) || streamName.equals(ImageStreams.IMG_PREVIEW.name()); + return streamName.equals(ImageStreams.IMG_FULL.name()) || streamName.equals(ImageStreams.IMG_PREVIEW.name()); } - @Override @@ -249,9 +254,9 @@ public void processAccessLog(final ReportedAction repAction, final StatisticsAcc String sql = records.toString(); new JDBCQueryTemplate(this.connectionProvider.get()) { - - private int record_id=-1; - + + private int record_id = -1; + @Override public boolean handleRow(ResultSet rs, List returnsList) throws SQLException { int record_id = rs.getInt("record_id"); @@ -262,7 +267,7 @@ public boolean handleRow(ResultSet rs, List returnsList) throws SQLExcep processDetail(rs); } this.record_id = record_id; - + return super.handleRow(rs, returnsList); } @@ -274,7 +279,8 @@ private void processDetail(ResultSet rs) throws SQLException { String lang = rs.getString("dlang"); String title = rs.getString("dtitle"); - Map record = new HashMap(); { + Map record = new HashMap(); + { record.put("model", model); record.put("pid", pid); record.put("issued_date", issuedDate); @@ -283,7 +289,7 @@ private void processDetail(ResultSet rs) throws SQLException { record.put("title", title); } sup.processDetailRecord(record); - } + } private void processMaster(ResultSet rs) throws SQLException { String pid = rs.getString("spid"); @@ -295,21 +301,22 @@ private void processMaster(ResultSet rs) throws SQLException { String sessionId = rs.getString("ssession_id"); - Map record = new HashMap(); { + Map record = new HashMap(); + { record.put("pid", pid); record.put("date", d); record.put("remote_ip_address", remote); record.put("user", user); - record.put("action",action); + record.put("action", action); record.put("session_id", sessionId); record.put("requested_url", requestedUrl); } sup.processMainRecord(record); - + } - + }.executeQuery(sql); - + } @Override @@ -326,13 +333,13 @@ public StatisticReport getReportById(String reportId) { } public static String disectedURL(Provider requestProvider) { - String url = requestProvider.get().getRequestURL().toString() +"?"+requestProvider.get().getQueryString(); + String url = requestProvider.get().getRequestURL().toString() + "?" + requestProvider.get().getQueryString(); return url; } - - + + public static class InsertRecord extends JDBCCommand { - + private LoggedUsersSingleton loggedUserSingleton; private Provider requestProvider; private Provider userProvider; @@ -348,7 +355,7 @@ public static class InsertRecord extends JDBCCommand { private String[] dnnt_labels; - public InsertRecord(String pid,LoggedUsersSingleton loggedUserSingleton, Provider requestProvider, Provider userProvider, ReportedAction action, boolean dnnt, boolean providedByDnnt, Map evaulateMap, Map userSesionAttributes, String dbversion, List dnnt_labels) { + public InsertRecord(String pid, LoggedUsersSingleton loggedUserSingleton, Provider requestProvider, Provider userProvider, ReportedAction action, boolean dnnt, boolean providedByDnnt, Map evaulateMap, Map userSesionAttributes, String dbversion, List dnnt_labels) { super(); this.loggedUserSingleton = loggedUserSingleton; this.requestProvider = requestProvider; @@ -368,41 +375,38 @@ public InsertRecord(String pid,LoggedUsersSingleton loggedUserSingleton, Provide } - - public Object executeJDBCCommand(Connection con) throws SQLException { Map previousResult = (Map) getPreviousResult(); if (previousResult == null) previousResult = new HashMap(); - + final StringTemplate statRecord = stGroup.getInstanceOf("insertStatisticRecord"); String sessionId = requestProvider.get().getSession().getId(); boolean logged = loggedUserSingleton.isLoggedUser(requestProvider); Object user = logged ? userProvider.get().getLoginname() : new JDBCUpdateTemplate.NullObject(String.class); String url = disectedURL(requestProvider); //requestProvider.get().getRequestURL().toString() +"?"+requestProvider.get().getQueryString(); - int record_id = new JDBCUpdateTemplate(con, false) - .executeUpdate( - statRecord.toString(), - pid, - new java.sql.Timestamp(System.currentTimeMillis()), - requestProvider.get().getRemoteAddr(), - user, - url, - action != null ? action.name() : ReportedAction.READ.name() , - sessionId, - dnnt, - providedByDnnt, - evaluateMap != null && !evaluateMap.isEmpty() ? new JSONObject(evaluateMap).toString() : new JDBCUpdateTemplate.NullObject(String.class), - userSesionAttributes != null && !userSesionAttributes.isEmpty() ? new JSONObject(userSesionAttributes).toString() :new JDBCUpdateTemplate.NullObject(String.class), - dbversion, - dnnt_labels - ); + int record_id = new JDBCUpdateTemplate(con, false) + .executeUpdate( + statRecord.toString(), + pid, + new java.sql.Timestamp(System.currentTimeMillis()), + requestProvider.get().getRemoteAddr(), + user, + url, + action != null ? action.name() : ReportedAction.READ.name(), + sessionId, + dnnt, + providedByDnnt, + evaluateMap != null && !evaluateMap.isEmpty() ? new JSONObject(evaluateMap).toString() : new JDBCUpdateTemplate.NullObject(String.class), + userSesionAttributes != null && !userSesionAttributes.isEmpty() ? new JSONObject(userSesionAttributes).toString() : new JDBCUpdateTemplate.NullObject(String.class), + dbversion, + dnnt_labels + ); previousResult.put("record_id", new Integer(record_id)); - return previousResult; } } @@ -416,13 +420,13 @@ public static class InsertDetail extends JDBCCommand { private Object date = null; private Object solrDate = null; private int pathIndex = 0; - private Object rights=null; + private Object rights = null; private String[] isbn = new String[0]; - private String[] issn = new String[0]; - private String[] ccnb = new String[0]; + private String[] issn = new String[0]; + private String[] ccnb = new String[0]; - public InsertDetail(String detailPid, String kModel,Object rights ,Object date, Object solrDate, Object language, Object title, Map> modsIdents, int pathIndex) { + public InsertDetail(String detailPid, String kModel, Object rights, Object date, Object solrDate, Object language, Object title, Map> modsIdents, int pathIndex) { super(); this.detailPid = detailPid; this.kModel = kModel; @@ -434,13 +438,13 @@ public InsertDetail(String detailPid, String kModel,Object rights ,Object date, this.solrDate = solrDate; if (modsIdents.containsKey(ISBN_MODS_KEY)) { - isbn = modsIdents.get(ISBN_MODS_KEY).toArray(new String[ modsIdents.get(ISBN_MODS_KEY).size() ]); + isbn = modsIdents.get(ISBN_MODS_KEY).toArray(new String[modsIdents.get(ISBN_MODS_KEY).size()]); } if (modsIdents.containsKey(ISSN_MODS_KEY)) { - issn = modsIdents.get(ISSN_MODS_KEY).toArray(new String[ modsIdents.get(ISSN_MODS_KEY).size() ]); + issn = modsIdents.get(ISSN_MODS_KEY).toArray(new String[modsIdents.get(ISSN_MODS_KEY).size()]); } if (modsIdents.containsKey(CCNB_MODS_KEY)) { - ccnb = modsIdents.get(CCNB_MODS_KEY).toArray(new String[ modsIdents.get(CCNB_MODS_KEY).size() ]); + ccnb = modsIdents.get(CCNB_MODS_KEY).toArray(new String[modsIdents.get(CCNB_MODS_KEY).size()]); } } @@ -453,17 +457,17 @@ public Object executeJDBCCommand(Connection con) throws SQLException { String sql = detail.toString(); int record_id = previousResult.get("record_id"); - int detail_id = new JDBCUpdateTemplate(con, false) - .executeUpdate(sql, detailPid, kModel, date, solrDate, rights , language, title, pathIndex,record_id, issn, isbn, ccnb); - + int detail_id = new JDBCUpdateTemplate(con, false) + .executeUpdate(sql, detailPid, kModel, date, solrDate, rights, language, title, pathIndex, record_id, issn, isbn, ccnb); + previousResult.put("detail_id", detail_id); - + return previousResult; - } + } } public static class InsertAuthor extends JDBCCommand { - + private String authorName; public InsertAuthor(String authorName) { @@ -481,10 +485,10 @@ public Object executeJDBCCommand(Connection con) throws SQLException { String sql = detail.toString(); int record_id = previousResult.get("record_id"); int detail_id = previousResult.get("detail_id"); - + new JDBCUpdateTemplate(con, false) - .executeUpdate(sql, this.authorName, detail_id, record_id); - + .executeUpdate(sql, this.authorName, detail_id, record_id); + return previousResult; } } @@ -516,13 +520,48 @@ public Object executeJDBCCommand(Connection con) throws SQLException { return previousResult; } } - + public static StringTemplateGroup stGroup; + static { InputStream is = DatabaseStatisticsAccessLogImpl.class.getResourceAsStream("res/statistics.stg"); stGroup = new StringTemplateGroup(new InputStreamReader(is), DefaultTemplateLexer.class); } + + + @Override + public int cleanData(java.util.Date dateFrom, java.util.Date dateTo) throws IOException{ + throw new UnsupportedOperationException("unsupported operation"); + } + + @Override + public void refresh() throws IOException { + try { + LOGGER.info("Refreshing materialized views (lang)"); + InputStream langIs = StatisticDbInitializer.class.getResourceAsStream("res/refreshlang.sql"); + JDBCUpdateTemplate langTemplate = new JDBCUpdateTemplate(connectionProvider.get(), true); + langTemplate.setUseReturningKeys(false); + langTemplate.executeUpdate(IOUtils.readAsString(langIs, Charset.forName("UTF-8"), true)); + + + LOGGER.info("Refreshing materialized views (authors)"); + InputStream authorsIs = StatisticDbInitializer.class.getResourceAsStream("res/refreshauthors.sql"); + JDBCUpdateTemplate authorsTemplate = new JDBCUpdateTemplate(connectionProvider.get(), true); + authorsTemplate.setUseReturningKeys(false); + authorsTemplate.executeUpdate(IOUtils.readAsString(authorsIs, Charset.forName("UTF-8"), true)); + + LOGGER.info("Refreshing materialized views (models)"); + InputStream modelsIs = StatisticDbInitializer.class.getResourceAsStream("res/refreshmodels.sql"); + JDBCUpdateTemplate modelsTemplate = new JDBCUpdateTemplate(connectionProvider.get(), true); + modelsTemplate.setUseReturningKeys(false); + modelsTemplate.executeUpdate(IOUtils.readAsString(modelsIs, Charset.forName("UTF-8"), true)); + + } catch (SQLException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } + + } } diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/accesslogs/dnnt/DNNTStatisticsAccessLogImpl.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/accesslogs/dnnt/DNNTStatisticsAccessLogImpl.java index 08e30d4910..978607acbd 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/accesslogs/dnnt/DNNTStatisticsAccessLogImpl.java +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/accesslogs/dnnt/DNNTStatisticsAccessLogImpl.java @@ -41,6 +41,7 @@ public class DNNTStatisticsAccessLogImpl extends AbstractStatisticsAccessLog { + public static final String PUBLISHERS_KEY = "publishers"; public static final String DNNT_LABELS_KEY = "dnnt-labels"; public static final String PIDS_PATH_KEY = "pids_path"; @@ -80,8 +81,8 @@ public void reportAccess(String pid, String streamName) throws IOException { ObjectPidsPath[] paths = this.solrAccess.getPidPaths(null, solrDoc); ObjectModelsPath[] mpaths = this.solrAccess.getModelPaths(solrDoc); - String rootTitle = SElemUtils.selem("str", "root.title", solrDoc); - String rootPid = SElemUtils.selem("str", "root.pid", solrDoc); + String rootTitle = SElemUtils.selem("str", "root.title", solrDoc); + String rootPid = SElemUtils.selem("str", "root.pid", solrDoc); String dctitle = SElemUtils.selem("str", "title.search", solrDoc); String solrDate = SElemUtils.selem("str", "date.str", solrDoc); //String dnnt = SElemUtils.selem("bool", "dnnt", solrDoc); @@ -100,7 +101,7 @@ public void reportAccess(String pid, String streamName) throws IOException { } - public static List solrAuthors(String rootPid, SolrAccess solrAccess) throws IOException { + public static List solrAuthors(String rootPid, SolrAccess solrAccess) throws IOException { List sAuthors = new ArrayList<>(); if (rootPid != null) { Document rootSolrDoc = solrAccess.getSolrDataByPid(rootPid); @@ -121,7 +122,7 @@ public boolean acceptElement(Element element) { return sAuthors; } - public static List dcPublishers(ObjectPidsPath[] paths, FedoraAccess fedoraAccess) throws IOException { + public static List dcPublishers(ObjectPidsPath[] paths, FedoraAccess fedoraAccess) throws IOException { List dcPublishers = new ArrayList<>(); for (int i = 0, ll = paths.length; i < ll; i++) { if (paths[i].contains(SpecialObjects.REPOSITORY.getPid())) { @@ -131,12 +132,18 @@ public static List dcPublishers(ObjectPidsPath[] paths, FedoraAccess fe String[] pathFromLeafToRoot = paths[i].getPathFromLeafToRoot(); for (int j = 0; j < pathFromLeafToRoot.length; j++) { final String detailPid = pathFromLeafToRoot[j]; - Document document = fedoraAccess.getDC(detailPid); - - List collected = Arrays.stream(DCUtils.publishersFromDC(document)).map(it -> { - return it.replaceAll("\\r?\\n", " "); - }).collect(Collectors.toList()); - dcPublishers.addAll(collected); + Document dc = null; + try { + dc = fedoraAccess.getDC(detailPid); + } catch (IOException e) { + LOGGER.warning("datastream DC not found for " + detailPid + ", ignoring statistics"); + } + if (dc != null) { + List collected = Arrays.stream(DCUtils.publishersFromDC(dc)).map(it -> { + return it.replaceAll("\\r?\\n", " "); + }).collect(Collectors.toList()); + dcPublishers.addAll(collected); + } } } return dcPublishers; @@ -183,12 +190,12 @@ public static String findModsDate(ObjectPidsPath[] paths, FedoraAccess fedoraAcc public static Map> identifiers(ObjectPidsPath[] paths, FedoraAccess fedoraAccess) throws IOException { try { - Map> retmap = new HashMap<>(); + Map> retmap = new HashMap<>(); for (ObjectPidsPath path : paths) { String[] pathFromLeafToRoot = path.getPathFromLeafToRoot(); for (String detailPid : pathFromLeafToRoot) { Map> map = ModsUtils.identifiersFromMods(fedoraAccess.getBiblioMods(detailPid)); - Arrays.asList(ISBN_MODS_KEY,ISSN_MODS_KEY,CCNB_MODS_KEY).stream().forEach(key-> { + Arrays.asList(ISBN_MODS_KEY, ISSN_MODS_KEY, CCNB_MODS_KEY).stream().forEach(key -> { if (map.containsKey(key)) { if (retmap.containsKey(key)) { retmap.get(key).addAll(map.get(key)); @@ -234,7 +241,7 @@ public void log(String pid, String rootTitle, String dcTitle, String solrDate, S ObjectModelsPath[] mpaths, Map> identifiers, List labels) throws IOException { User user = this.userProvider.get(); RightsReturnObject rightsReturnObject = CriteriaDNNTUtils.currentThreadReturnObject.get(); - boolean providedByDnnt = rightsReturnObject != null ? CriteriaDNNTUtils.allowedByReadDNNTFlagRight(rightsReturnObject) : false; + boolean providedByDnnt = rightsReturnObject != null ? CriteriaDNNTUtils.allowedByReadDNNTFlagRight(rightsReturnObject) : false; // store json object JSONObject jObject = toJSON(pid, rootTitle, dcTitle, @@ -246,7 +253,7 @@ public void log(String pid, String rootTitle, String dcTitle, String solrDate, S dnntFlag, providedByDnnt, policy, - rightsReturnObject.getEvaluateInfoMap(), + rightsReturnObject != null ? rightsReturnObject.getEvaluateInfoMap(): new HashMap(), user.getSessionAttributes(), dcAuthors, dcPublishers, @@ -259,116 +266,133 @@ public void log(String pid, String rootTitle, String dcTitle, String solrDate, S DNNTStatisticsAccessLogImpl.KRAMERIUS_LOGGER_FOR_KIBANA.log(Level.INFO, jObject.toString()); } - public static JSONObject toJSON(String pid, - String rootTitle, - String dcTitle, - String remoteAddr, - String username, - String email, - String solrDate, - String modsDate, - String dnntFlag, - boolean providedByDnnt, - String policy, - Map rightEvaluationAttribute, - Map sessionAttributes, - List dcAuthors, - List dcPublishers, - ObjectPidsPath[] paths, - ObjectModelsPath[] mpaths, - Map> identifiers, - List labels - ) throws IOException { + public static JSONObject toJSON(String pid, + String rootTitle, + String dcTitle, + String remoteAddr, + String username, + String email, + String solrDate, + String modsDate, + String dnntFlag, + boolean providedByDnnt, + String policy, + Map rightEvaluationAttribute, + Map sessionAttributes, + List dcAuthors, + List dcPublishers, + ObjectPidsPath[] paths, + ObjectModelsPath[] mpaths, + Map> identifiers, + List labels + ) throws IOException { LocalDateTime date = LocalDateTime.now(); String timestamp = date.format(DateTimeFormatter.ISO_DATE_TIME); JSONObject jObject = new JSONObject(); - jObject.put("pid",pid); - jObject.put("remoteAddr",remoteAddr); - jObject.put("username",username); - jObject.put("email",email); + jObject.put("pid", pid); + jObject.put("remoteAddr", remoteAddr); + jObject.put("username", username); + jObject.put("email", email); - jObject.put("rootTitle",rootTitle); - jObject.put("dcTitle",dcTitle); + jObject.put("rootTitle", rootTitle); + jObject.put("dcTitle", dcTitle); - if (dnntFlag != null ) jObject.put(DNNT_KEY, dnntFlag.trim().toLowerCase().equals("true")); + if (dnntFlag != null) jObject.put(DNNT_KEY, dnntFlag.trim().toLowerCase().equals("true")); // info from criteriums - rightEvaluationAttribute.keySet().stream().forEach(key->{ jObject.put(key, rightEvaluationAttribute.get(key)); }); + rightEvaluationAttribute.keySet().stream().forEach(key -> { + jObject.put(key, rightEvaluationAttribute.get(key)); + }); jObject.put(PROVIDED_BY_DNNT_KEY, providedByDnnt); jObject.put(POLICY_KEY, policy); - if (solrDate != null) jObject.put(SOLR_DATE_KEY, solrDate); + if (solrDate != null) jObject.put(SOLR_DATE_KEY, solrDate); if (modsDate != null) jObject.put("publishedDate", modsDate); - jObject.put("date",timestamp); + jObject.put("date", timestamp); - sessionAttributes.keySet().stream().forEach(key->{ jObject.put(key, sessionAttributes.get(key)); }); + sessionAttributes.keySet().stream().forEach(key -> { + jObject.put(key, sessionAttributes.get(key)); + }); if (!dcAuthors.isEmpty()) { JSONArray authorsArray = new JSONArray(); - for (int i=0,ll=dcAuthors.size();i 0) { String[] pathFromRootToLeaf = paths[0].getPathFromRootToLeaf(); if (pathFromRootToLeaf.length > 0) { - jObject.put(ROOT_PID_KEY,pathFromRootToLeaf[0]); + jObject.put(ROOT_PID_KEY, pathFromRootToLeaf[0]); } } if (mpaths.length > 0) { String[] mpathFromRootToLeaf = mpaths[0].getPathFromRootToLeaf(); if (mpathFromRootToLeaf.length > 0) { - jObject.put(ROOT_MODEL_KEY,mpathFromRootToLeaf[0]); + jObject.put(ROOT_MODEL_KEY, mpathFromRootToLeaf[0]); } } - identifiers.keySet().forEach(key-> {jObject.put(key, identifiers.get(key).stream().filter(Objects::nonNull).distinct().collect(Collectors.toList()));}); + identifiers.keySet().forEach(key -> { + jObject.put(key, identifiers.get(key).stream().filter(Objects::nonNull).distinct().collect(Collectors.toList())); + }); return jObject; } - private String getDate(DNNTStatisticsDateFormat dateFormat, String publishedDate) { + private String getDate(DNNTStatisticsDateFormat dateFormat, String publishedDate) { if (dateFormat != null && publishedDate != null) return dateFormat.format(publishedDate); else return null; } + + + @Override + public int cleanData(Date dateFrom, Date dateTo) { + throw new UnsupportedOperationException("unsupported operation"); + } + + + @Override + public void refresh() throws IOException { + } } diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/filters/DateFilter.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/filters/DateFilter.java index e58fec9e68..2cd0878e27 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/filters/DateFilter.java +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/filters/DateFilter.java @@ -1,25 +1,57 @@ package cz.incad.kramerius.statistics.filters; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Date; import java.util.logging.Level; import java.util.logging.Logger; +import cz.incad.kramerius.statistics.StatisticReport; + /** * Represents simple date filter (date_from - date_to) * @author pstastny */ public class DateFilter implements StatisticsFilter { + public static final Logger LOGGER = Logger.getLogger(DateFilter.class.getName()); + private SimpleDateFormat inputFormat = StatisticReport.DATE_FORMAT; + private SimpleDateFormat outputFormat = StatisticReport.SOLR_DATE_FORMAT; + private String fromDate; private String toDate; + /** * Returns date from * @return */ public String getFromDate() { - return fromDate; + if(this.fromDate != null) { + return getFormattedDate(this.fromDate); + } else return null; + } + + + public String getRawFromDate() { + return this.fromDate; + } + + + private String getFormattedDate(String fDate) { + if (this.inputFormat != null && this.outputFormat !=null) { + try { + Date parsed = this.inputFormat.parse(fDate); + return this.outputFormat.format(parsed); + } catch (ParseException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + return fDate; + } + } else { + return fDate; + } } /** @@ -30,7 +62,7 @@ public void setFromDate(String fromDate) { if (fromDate != null && (!fromDate.trim().equals(""))) { this.fromDate = fromDate; } else { - LOGGER.severe("bad value"); + //LOGGER.severe("bad value"); } } @@ -39,9 +71,17 @@ public void setFromDate(String fromDate) { * @return */ public String getToDate() { - return toDate; + if (this.toDate != null) { + return getFormattedDate(toDate); + } else { + return null; + } } + public String getRawToDate() { + return this.toDate; + } + /** * Sets date to * @param toDate @@ -50,8 +90,25 @@ public void setToDate(String toDate) { if (toDate != null && (!toDate.trim().equals(""))) { this.toDate = toDate; } else { - LOGGER.severe("bad value"); + //LOGGER.severe("bad value"); } } + + + public SimpleDateFormat getInputFormat() { + return inputFormat; + } + + public void setInputFormat(SimpleDateFormat inputFormat) { + this.inputFormat = inputFormat; + } + + public SimpleDateFormat getOutputFormat() { + return outputFormat; + } + + public void setOutputFormat(SimpleDateFormat outputFormat) { + this.outputFormat = outputFormat; + } } diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/filters/IdentifiersFilter.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/filters/IdentifiersFilter.java new file mode 100644 index 0000000000..529ea6df6f --- /dev/null +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/filters/IdentifiersFilter.java @@ -0,0 +1,14 @@ +package cz.incad.kramerius.statistics.filters; + +public class IdentifiersFilter implements StatisticsFilter { + + private String identifier; + + public String getIdentifier() { + return identifier; + } + + public void setIdentifier(String identifier) { + this.identifier = identifier; + } +} diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/filters/LicenseFilter.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/filters/LicenseFilter.java new file mode 100644 index 0000000000..5a1eef8314 --- /dev/null +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/filters/LicenseFilter.java @@ -0,0 +1,20 @@ +package cz.incad.kramerius.statistics.filters; + +import java.util.logging.Logger; + +public class LicenseFilter implements StatisticsFilter { + + public static final Logger LOGGER = Logger.getLogger(LicenseFilter.class.getName()); + + private String licence; + + public LicenseFilter(String licence) { + super(); + this.licence = licence; + } + + public String getLicence() { + return licence; + } + +} diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/formatters/report/nkp/NKPJSONFormatter.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/formatters/report/nkp/NKPJSONFormatter.java deleted file mode 100644 index e25c027b71..0000000000 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/formatters/report/nkp/NKPJSONFormatter.java +++ /dev/null @@ -1,68 +0,0 @@ -package cz.incad.kramerius.statistics.formatters.report.nkp; - -import com.google.inject.Inject; -import com.google.inject.name.Named; -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.statistics.formatters.report.StatisticsReportFormatter; -import cz.incad.kramerius.statistics.impl.NKPLogReport; -import org.json.JSONObject; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; - -public class NKPJSONFormatter implements StatisticsReportFormatter { - - public static final Logger LOGGER = Logger.getLogger(NKPJSONFormatter.class.getName()); - - private OutputStream os; - - @Inject - @Named("rawFedoraAccess") - FedoraAccess fedoraAccess; - - - @Override - public String getMimeType() { - return JSON_MIME_TYPE; - } - - @Override - public String getFormat() { - return JSON_FORMAT; - } - - @Override - public String getReportId() { - return NKPLogReport.REPORT_ID; - } - - @Override - public void processReportRecord(Map record) { - try { - JSONObject object = new JSONObject(record); - this.os.write(object.toString().getBytes(DEFAULT_ENCODING)); - this.os.write("\n".getBytes(DEFAULT_ENCODING)); - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(),e); - } - } - - @Override - public void beforeProcess(OutputStream os) throws IOException { - this.os = os; - } - - @Override - public void afterProcess(OutputStream os) throws IOException { - this.os = null; - } - - @Override - public void addInfo(OutputStream os, String info) throws IOException { - this.os = os; - } -} diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/formatters/report/nkp/NKPXMLFormatter.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/formatters/report/nkp/NKPXMLFormatter.java deleted file mode 100644 index 9be9210b6a..0000000000 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/formatters/report/nkp/NKPXMLFormatter.java +++ /dev/null @@ -1,118 +0,0 @@ -package cz.incad.kramerius.statistics.formatters.report.nkp; - -import com.google.inject.Inject; -import com.google.inject.name.Named; -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.ObjectModelsPath; -import cz.incad.kramerius.ObjectPidsPath; -import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.statistics.accesslogs.dnnt.DNNTStatisticsAccessLogImpl; -import cz.incad.kramerius.statistics.accesslogs.utils.SElemUtils; -import cz.incad.kramerius.statistics.formatters.report.StatisticsReportFormatter; -import cz.incad.kramerius.statistics.formatters.utils.StringUtils; -import cz.incad.kramerius.statistics.impl.NKPLogReport; -import org.w3c.dom.Document; - -import java.io.IOException; -import java.io.OutputStream; -import java.io.UnsupportedEncodingException; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; - -public class NKPXMLFormatter implements StatisticsReportFormatter { - - public static final Logger LOGGER = Logger.getLogger(NKPXMLFormatter.class.getName()); - - private OutputStream os; - - @Inject - @Named("rawFedoraAccess") - FedoraAccess fedoraAccess; - - - @Override - public String getMimeType() { - return XML_MIME_TYPE; - } - - @Override - public String getFormat() { - return XML_FORMAT; - } - - - -// @Override -// public void beforeProcess(HttpServletResponse response) throws IOException { -// this.os.write("\n".getBytes(DEFAULT_ENCODING)); -// } - -// @Override -// public void afterProcess(HttpServletResponse response) throws IOException { -// this.os.write("\n".getBytes(DEFAULT_ENCODING)); -// this.os = null; -// } - - @Override - public void processReportRecord(Map record) { - try { - - String pid = (String) record.get("pid"); - - StringBuilder builder = new StringBuilder("\n"); - - builder.append("\t").append(record.get("pid")).append("\n"); - builder.append("\t").append(record.get("date")).append("\n"); - builder.append("\t").append(record.get("user")).append("\n"); - - builder.append("\t").append(record.get("dnnt")).append("\n"); - builder.append("\t").append(record.get("providedByDnnt")).append("\n"); - builder.append("\t").append(StringUtils.nullify((String)record.get("model"))).append("\n"); - builder.append("\t").append(StringUtils.nullify((String)record.get("rights"))).append("\n"); - - - builder.append("\n"); - - - this.os.write(builder.toString().getBytes(DEFAULT_ENCODING)); - } catch (UnsupportedEncodingException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } catch (IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } - } - - @Override - public String getReportId() { - return NKPLogReport.REPORT_ID; - } - -// @Override -// public void addInfo(HttpServletResponse response, String info) throws IOException { -// this.os = response.getOutputStream(); -// String text = "Report pro nkp "; -// String comment = ""; -// this.os.write(comment.getBytes(DEFAULT_ENCODING)); -// } - - @Override - public void beforeProcess(OutputStream os) throws IOException { - this.os.write("\n".getBytes(DEFAULT_ENCODING)); - } - - @Override - public void afterProcess(OutputStream os) throws IOException { - this.os.write("\n".getBytes(DEFAULT_ENCODING)); - this.os = null; - } - - @Override - public void addInfo(OutputStream os, String info) throws IOException { - this.os = os; - String text = "Report pro nkp "; - String comment = ""; - this.os.write(comment.getBytes(DEFAULT_ENCODING)); - } -} diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/AbstractStatisticsReport.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/AbstractStatisticsReport.java new file mode 100644 index 0000000000..aaa3c9f664 --- /dev/null +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/AbstractStatisticsReport.java @@ -0,0 +1,41 @@ +package cz.incad.kramerius.statistics.impl; + +import cz.incad.kramerius.statistics.StatisticReport; +import cz.incad.kramerius.statistics.filters.DateFilter; +import cz.incad.kramerius.statistics.filters.IdentifiersFilter; +import cz.incad.kramerius.statistics.filters.LicenseFilter; +import cz.incad.kramerius.statistics.filters.ModelFilter; +import cz.incad.kramerius.statistics.filters.StatisticsFiltersContainer; +import cz.incad.kramerius.statistics.utils.ReportUtils; +import cz.incad.kramerius.utils.conf.KConfiguration; + +public abstract class AbstractStatisticsReport implements StatisticReport { + + protected String logsEndpoint() { + String loggerPoint = KConfiguration.getInstance().getProperty("k7.log.solr.point","http://localhost:8983/solr/logs"); + String selectEndpoint = loggerPoint + (loggerPoint.endsWith("/") ? "" : "/" ) +""; + return selectEndpoint; + } + + protected void applyFilters(StatisticsFiltersContainer filters, StringBuilder builder) { + DateFilter dateFilter = filters.getFilter(DateFilter.class); + LicenseFilter licFilter = filters.getFilter(LicenseFilter.class); + IdentifiersFilter idFilter = filters.getFilter(IdentifiersFilter.class); + ModelFilter modelFilter = filters.getFilter(ModelFilter.class); + + //StringBuilder builder = new StringBuilder("q=*"); + ReportUtils.enhanceLicense(builder, licFilter); + ReportUtils.enhanceDateFilter(builder, dateFilter); + ReportUtils.enhanceIdentifiers(builder, idFilter); + ReportUtils.enhanceModelFilter(builder, modelFilter); + + //return builder; + } + + @Override + public boolean convertToObject() { + return false; + } + + +} diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/AnnualStatisticsReport.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/AnnualStatisticsReport.java index d5363f8f17..4e17a50014 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/AnnualStatisticsReport.java +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/AnnualStatisticsReport.java @@ -14,7 +14,8 @@ import java.util.*; import java.util.logging.Logger; -public class AnnualStatisticsReport implements StatisticReport { +//TODO: Vyhodit +public class AnnualStatisticsReport extends AbstractStatisticsReport implements StatisticReport { @@ -35,7 +36,7 @@ public List> getReportPage(ReportedAction reportedAction, St } @Override - public List getOptionalValues() { + public List getOptionalValues(StatisticsFiltersContainer filters) { return new ArrayList<>(); } @@ -44,45 +45,24 @@ public String getReportId() { return REPORT_ID; } - @Override - public void prepareViews(ReportedAction action, StatisticsFiltersContainer container) throws StatisticsReportException { - MultimodelFilter multimodel = container.getFilter(MultimodelFilter.class); - List models = multimodel.getModels(); - for (String model : models) { - ModelFilter modelFilter = new ModelFilter(); - modelFilter.setModel(model); - StatisticsFilter[] filters = new StatisticsFilter[] { - modelFilter, - getDateFilter(container.getFilter(AnnualYearFilter.class)), - container.getFilter(IPAddressFilter.class), - new VisibilityFilter() - }; - StatisticsFiltersContainer subcontainer = new StatisticsFiltersContainer(filters); - ModelStatisticReport report = new ModelStatisticReport(); - report.connectionProvider = connectionProvider; - report.prepareViews(action, subcontainer); - } - } - @Override public void processAccessLog(ReportedAction action, StatisticsReportSupport sup, StatisticsFiltersContainer container) throws StatisticsReportException { - MultimodelFilter multimodel = container.getFilter(MultimodelFilter.class); - List models = multimodel.getModels(); - for (String model : models) { - ModelFilter modelFilter = new ModelFilter(); - modelFilter.setModel(model); - StatisticsFilter[] filters = new StatisticsFilter[]{ - modelFilter, - getDateFilter(container.getFilter(AnnualYearFilter.class)), - container.getFilter(IPAddressFilter.class), - container.getFilter(VisibilityFilter.class), - container.getFilter(UniqueIPAddressesFilter.class) - }; - ModelStatisticReport report = new ModelStatisticReport(); - report.connectionProvider = connectionProvider; - report.processAccessLog(action, sup, new StatisticsFiltersContainer(filters)); - - } +// MultimodelFilter multimodel = container.getFilter(MultimodelFilter.class); +// List models = multimodel.getModels(); +// for (String model : models) { +// ModelFilter modelFilter = new ModelFilter(); +// modelFilter.setModel(model); +// StatisticsFilter[] filters = new StatisticsFilter[]{ +// modelFilter, +// getDateFilter(container.getFilter(AnnualYearFilter.class)), +// container.getFilter(IPAddressFilter.class), +// container.getFilter(VisibilityFilter.class), +// container.getFilter(UniqueIPAddressesFilter.class) +// }; +// ModelStatisticReport report = new ModelStatisticReport(); +// report.connectionProvider = connectionProvider; +// report.processAccessLog(action, sup, new StatisticsFiltersContainer(filters)); +// } } public static StatisticsFilter getDateFilter(AnnualYearFilter afilter) { @@ -114,8 +94,13 @@ public static StatisticsFilter getDateFilter(AnnualYearFilter afilter) { return filter; } - @Override - public boolean verifyFilters(ReportedAction action, StatisticsFiltersContainer container) { - return true; - } + @Override + public List verifyFilters(ReportedAction action, StatisticsFiltersContainer container) { + List list = new ArrayList<>(); + AnnualYearFilter modelFilter = container.getFilter(AnnualYearFilter.class); + if (modelFilter.getAnnualYear() == null) { + list.add("annualYeer is mandatory"); + } + return list; + } } diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/AuthorReport.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/AuthorReport.java index ef71b45d67..b6a5d517bc 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/AuthorReport.java +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/AuthorReport.java @@ -19,6 +19,8 @@ */ package cz.incad.kramerius.statistics.impl; +import java.io.IOException; +import java.io.InputStream; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -32,6 +34,7 @@ import cz.incad.kramerius.statistics.accesslogs.database.DatabaseStatisticsAccessLogImpl; import org.antlr.stringtemplate.StringTemplate; +import org.apache.commons.io.IOUtils; import com.google.inject.Inject; import com.google.inject.Provider; @@ -43,16 +46,21 @@ import cz.incad.kramerius.statistics.StatisticsReportSupport; import cz.incad.kramerius.statistics.filters.DateFilter; import cz.incad.kramerius.statistics.filters.IPAddressFilter; +import cz.incad.kramerius.statistics.filters.IdentifiersFilter; +import cz.incad.kramerius.statistics.filters.LicenseFilter; import cz.incad.kramerius.statistics.filters.StatisticsFiltersContainer; import cz.incad.kramerius.statistics.filters.UniqueIPAddressesFilter; +import cz.incad.kramerius.statistics.filters.VisibilityFilter; +import cz.incad.kramerius.statistics.utils.ReportUtils; +import cz.incad.kramerius.utils.conf.KConfiguration; import cz.incad.kramerius.utils.database.JDBCQueryTemplate; import cz.incad.kramerius.utils.database.Offset; /** + * Poskytnuti autori * @author pavels - * */ -public class AuthorReport implements StatisticReport{ +public class AuthorReport extends AbstractStatisticsReport implements StatisticReport{ public static final Logger LOGGER = Logger.getLogger(AuthorReport.class.getName()); @@ -67,54 +75,33 @@ public class AuthorReport implements StatisticReport{ @Override public List> getReportPage(ReportedAction repAction,StatisticsFiltersContainer filters, Offset rOffset) { try { - DateFilter dateFilter = filters.getFilter(DateFilter.class); - IPAddressFilter ipFilter = filters.getFilter(IPAddressFilter.class); - UniqueIPAddressesFilter uniqueIPFilter = filters.getFilter(UniqueIPAddressesFilter.class); + String selectEndpoint = super.logsEndpoint(); - Boolean isUniqueSelected = uniqueIPFilter.getUniqueIPAddresses(); - final StringTemplate statRecord; + StringBuilder builder = new StringBuilder("q=*"); + super.applyFilters(filters, builder); - if (isUniqueSelected == false) { - statRecord = DatabaseStatisticsAccessLogImpl.stGroup - .getInstanceOf("selectAuthorReport"); - } - else { - statRecord = DatabaseStatisticsAccessLogImpl.stGroup - .getInstanceOf("selectAuthorReportUnique"); - } + String facetField = "authors"; + builder.append(String.format("&rows=0&facet=true&facet.mincount=1&facet.field=%s", facetField)); + InputStream iStream = cz.incad.kramerius.utils.solr.SolrUtils.requestWithSelectReturningStream(selectEndpoint, builder.toString(), "json"); + String string = IOUtils.toString(iStream, "UTF-8"); + + List> authors = new ArrayList<>(); + ReportUtils.facetIterate(facetField, string, p-> { + Map map = new HashMap(); + map.put(COUNT_KEY, p.getValue()); + map.put(AUTHOR_NAME_KEY, p.getKey()); + authors.add(map); + }); - statRecord.setAttribute("action", repAction != null ? repAction.name() : null); - statRecord.setAttribute("paging", true); - statRecord.setAttribute("fromDefined", dateFilter.getFromDate() != null); - statRecord.setAttribute("toDefined", dateFilter.getToDate() != null); - statRecord.setAttribute("ipaddr", ipFilter.getIpAddress()); - - @SuppressWarnings("rawtypes") - List params = StatisticUtils.jdbcParams(dateFilter, rOffset); - String sql = statRecord.toString(); - Connection conn = connectionProvider.get(); - List> auths = new JDBCQueryTemplate>(conn) { - - @Override - public boolean handleRow(ResultSet rs, List> returnsList) throws SQLException { - Map map = new HashMap(); - map.put(COUNT_KEY, rs.getInt("count")); - map.put(AUTHOR_NAME_KEY, rs.getString("author_name")); - returnsList.add(map); - return super.handleRow(rs, returnsList); - } - }.executeQuery(sql.toString(), params.toArray()); - - LOGGER.fine(String.format("Test statistics connection.isClosed() : %b", conn.isClosed())); - return auths; - } catch (ParseException | SQLException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return new ArrayList>(); + return authors; + } catch (IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + return new ArrayList>(); } } @Override - public List getOptionalValues() { + public List getOptionalValues(StatisticsFiltersContainer filters) { return new ArrayList(); } @@ -124,65 +111,47 @@ public String getReportId() { } - @Override - public void prepareViews(ReportedAction action, StatisticsFiltersContainer container) { - // TODO Auto-generated method prepareViews - } @Override public void processAccessLog(final ReportedAction repAction, final StatisticsReportSupport sup, StatisticsFiltersContainer filters) throws StatisticsReportException { try { - final DateFilter dateFilter = filters.getFilter(DateFilter.class); - IPAddressFilter ipFilter = filters.getFilter(IPAddressFilter.class); - UniqueIPAddressesFilter uniqueIPFilter = filters.getFilter(UniqueIPAddressesFilter.class); - - Boolean isUniqueSelected = uniqueIPFilter.getUniqueIPAddresses(); - final StringTemplate statRecord; + String selectEndpoint = super.logsEndpoint(); - if (isUniqueSelected) { - statRecord = DatabaseStatisticsAccessLogImpl.stGroup - .getInstanceOf("selectAuthorReportUnique"); - } - else { - statRecord = DatabaseStatisticsAccessLogImpl.stGroup - .getInstanceOf("selectAuthorReport"); - } + DateFilter dateFilter = filters.getFilter(DateFilter.class); + LicenseFilter licFilter = filters.getFilter(LicenseFilter.class); + IdentifiersFilter idFilter = filters.getFilter(IdentifiersFilter.class); + + StringBuilder builder = new StringBuilder("q=*"); + + ReportUtils.enhanceLicense(builder, licFilter); + ReportUtils.enhanceDateFilter(builder, dateFilter); + ReportUtils.enhanceIdentifiers(builder, idFilter); - statRecord.setAttribute("action", repAction != null ? repAction.name() : null); - statRecord.setAttribute("paging", false); - statRecord.setAttribute("fromDefined", dateFilter.getFromDate() != null); - statRecord.setAttribute("toDefined", dateFilter.getToDate() != null); - statRecord.setAttribute("ipaddr", ipFilter.getIpAddress()); - - @SuppressWarnings("rawtypes") - List params = StatisticUtils.jdbcParams(dateFilter); - - String sql = statRecord.toString(); - Connection conn = connectionProvider.get(); - new JDBCQueryTemplate>(conn) { - - @Override - public boolean handleRow(ResultSet rs, List> returnsList) throws SQLException { - Map map = new HashMap(); - map.put(COUNT_KEY, rs.getInt("count")); - map.put(AUTHOR_NAME_KEY, rs.getString("author_name")); - returnsList.add(map); - sup.processReportRecord(map); - return super.handleRow(rs, returnsList); - } - }.executeQuery(sql.toString(),params.toArray()); - LOGGER.fine(String.format("Test statistics connection.isClosed() : %b", conn.isClosed())); - } catch (ParseException e) { + String facetField = "authors"; + builder.append(String.format("&rows=0&facet=true&facet.mincount=1&facet.field=%s", facetField)); + InputStream iStream = cz.incad.kramerius.utils.solr.SolrUtils.requestWithSelectReturningStream(selectEndpoint, builder.toString(), "json"); + String string = IOUtils.toString(iStream, "UTF-8"); + + ReportUtils.facetIterate(facetField, string, p-> { + Map map = new HashMap(); + map.put(COUNT_KEY, p.getValue()); + map.put(AUTHOR_NAME_KEY, p.getKey()); + sup.processReportRecord(map); + }); + + } catch (IOException e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); throw new StatisticsReportException(e); - } catch (SQLException ex) { - Logger.getLogger(AuthorReport.class.getName()).log(Level.SEVERE, null, ex); } } - @Override - public boolean verifyFilters(ReportedAction action, StatisticsFiltersContainer container) { - return true; - } + @Override + public List verifyFilters(ReportedAction action, StatisticsFiltersContainer container) { + List list = new ArrayList<>(); + DateFilter dateFilter = container.getFilter(DateFilter.class); + VerificationUtils.dateVerification(list, dateFilter.getRawFromDate()); + VerificationUtils.dateVerification(list, dateFilter.getRawToDate()); + return list; + } } diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/LangReport.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/LangReport.java index a07cc78583..bfe2b57343 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/LangReport.java +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/LangReport.java @@ -19,40 +19,56 @@ */ package cz.incad.kramerius.statistics.impl; +import java.io.IOException; +import java.io.InputStream; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; import java.text.ParseException; import java.util.ArrayList; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; import cz.incad.kramerius.statistics.accesslogs.database.DatabaseStatisticsAccessLogImpl; + import org.antlr.stringtemplate.StringTemplate; +import org.apache.commons.io.IOUtils; +import org.json.JSONArray; +import org.json.JSONObject; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.name.Named; +import cz.incad.kramerius.SolrAccess; import cz.incad.kramerius.statistics.ReportedAction; import cz.incad.kramerius.statistics.StatisticReport; import cz.incad.kramerius.statistics.StatisticsReportException; import cz.incad.kramerius.statistics.StatisticsReportSupport; import cz.incad.kramerius.statistics.filters.DateFilter; import cz.incad.kramerius.statistics.filters.IPAddressFilter; +import cz.incad.kramerius.statistics.filters.IdentifiersFilter; +import cz.incad.kramerius.statistics.filters.LicenseFilter; import cz.incad.kramerius.statistics.filters.StatisticsFiltersContainer; import cz.incad.kramerius.statistics.filters.UniqueIPAddressesFilter; +import cz.incad.kramerius.statistics.filters.VisibilityFilter; +import cz.incad.kramerius.statistics.utils.ReportUtils; +import cz.incad.kramerius.utils.RESTHelper; +import cz.incad.kramerius.utils.conf.KConfiguration; import cz.incad.kramerius.utils.database.JDBCQueryTemplate; import cz.incad.kramerius.utils.database.Offset; /** + * Report poskytnutych jazyku * * @author pavels */ -public class LangReport implements StatisticReport{ +public class LangReport extends AbstractStatisticsReport implements StatisticReport{ public static final Logger LOGGER = Logger.getLogger(LangReport.class.getName()); @@ -62,62 +78,42 @@ public class LangReport implements StatisticReport{ @Named("kramerius4") Provider connectionProvider; + @Override public List> getReportPage(ReportedAction repAction, StatisticsFiltersContainer filters,Offset rOffset) { try { - DateFilter dateFilter = filters.getFilter(DateFilter.class); + String selectEndpint = super.logsEndpoint(); - IPAddressFilter ipFilter = filters.getFilter(IPAddressFilter.class); - - UniqueIPAddressesFilter uniqueIPFilter = filters.getFilter(UniqueIPAddressesFilter.class); + List> langs = new ArrayList<>(); - Boolean isUniqueSelected = uniqueIPFilter.getUniqueIPAddresses(); - final StringTemplate statRecord; - - if (isUniqueSelected == false) { - statRecord = DatabaseStatisticsAccessLogImpl.stGroup - .getInstanceOf("selectLangReport"); - } - else { - statRecord = DatabaseStatisticsAccessLogImpl.stGroup - .getInstanceOf("selectLangReportUnique"); - } - - statRecord.setAttribute("action", repAction != null ? repAction.name() : null); - statRecord.setAttribute("fromDefined", dateFilter.getFromDate() != null); - statRecord.setAttribute("toDefined", dateFilter.getToDate() != null); - statRecord.setAttribute("ipaddr", ipFilter.getIpAddress()); - - @SuppressWarnings("rawtypes") - List params = StatisticUtils.jdbcParams(dateFilter); - //statRecord.setAttribute("paging", true); - String sql = statRecord.toString(); - Connection conn = connectionProvider.get(); - List> langs = new JDBCQueryTemplate>(conn) { - - @Override - public boolean handleRow(ResultSet rs, List> returnsList) throws SQLException { - Map map = new HashMap(); - map.put("count", rs.getInt("count")); - map.put("lang", rs.getString("lang")); - returnsList.add(map); - return super.handleRow(rs, returnsList); - } - }.executeQuery(sql.toString(),params.toArray()); - LOGGER.fine(String.format("Test statistics connection.isClosed() : %b", conn.isClosed())); + StringBuilder builder = new StringBuilder("q=*"); +// ReportUtils.enhanceLicense(builder, licFilter); +// ReportUtils.enhanceDateFilter(builder, dateFilter); +// ReportUtils.enhanceIdentifiers(builder, idFilter); + super.applyFilters(filters,builder); + + String facetField = "langs"; + builder.append(String.format("&rows=0&facet=true&facet.mincount=1&facet.field=%s", facetField)); + InputStream iStream = cz.incad.kramerius.utils.solr.SolrUtils.requestWithSelectReturningStream(selectEndpint, builder.toString(), "json"); + String string = IOUtils.toString(iStream, "UTF-8"); + + ReportUtils.facetIterate(facetField, string, p-> { + Map map = new HashMap(); + map.put(COUNT_KEY, p.getValue()); + map.put(LANG_KEY, p.getKey()); + langs.add(map); + }); + return langs; - } catch (ParseException e) { + } catch (IOException e) { LOGGER.log(Level.SEVERE,e.getMessage(),e); return new ArrayList>(); - } catch (SQLException ex) { - Logger.getLogger(LangReport.class.getName()).log(Level.SEVERE, null, ex); - return new ArrayList>(); } } @Override - public List getOptionalValues() { + public List getOptionalValues(StatisticsFiltersContainer filters) { return new ArrayList(); } @@ -126,65 +122,45 @@ public String getReportId() { return REPORT_ID; } - - - - @Override - public void prepareViews(ReportedAction action, StatisticsFiltersContainer container) { - // TODO Auto-generated method stub - - } @Override public void processAccessLog(final ReportedAction repAction, final StatisticsReportSupport sup, - final StatisticsFiltersContainer container) throws StatisticsReportException { + final StatisticsFiltersContainer filters) throws StatisticsReportException { try { - DateFilter dateFilter = container.getFilter(DateFilter.class); - IPAddressFilter ipFilter = container.getFilter(IPAddressFilter.class); - UniqueIPAddressesFilter uniqueIPFilter = container.getFilter(UniqueIPAddressesFilter.class); - - Boolean isUniqueSelected = uniqueIPFilter.getUniqueIPAddresses(); - final StringTemplate statRecord; - - if (isUniqueSelected == false) { - statRecord = DatabaseStatisticsAccessLogImpl.stGroup - .getInstanceOf("selectLangReport"); - } - else { - statRecord = DatabaseStatisticsAccessLogImpl.stGroup - .getInstanceOf("selectLangReportUnique"); - } - statRecord.setAttribute("action", repAction != null ? repAction.name() : null); - statRecord.setAttribute("fromDefined", dateFilter.getFromDate() != null); - statRecord.setAttribute("toDefined", dateFilter.getToDate() != null); - statRecord.setAttribute("ipaddr", ipFilter.getIpAddress()); - statRecord.setAttribute("paging", false); - - - @SuppressWarnings("rawtypes") - List params = StatisticUtils.jdbcParams(dateFilter); - - String sql = statRecord.toString(); - Connection conn = connectionProvider.get(); - new JDBCQueryTemplate>(conn) { - @Override - public boolean handleRow(ResultSet rs, List> returnsList) throws SQLException { - Map map = new HashMap(); - map.put(COUNT_KEY, rs.getInt("count")); - map.put(LANG_KEY, rs.getString("lang")); - sup.processReportRecord(map); - return super.handleRow(rs, returnsList); - } - }.executeQuery(sql.toString(), params.toArray()); - conn.close(); + String selectEndpint = super.logsEndpoint(); + DateFilter dateFilter = filters.getFilter(DateFilter.class); + LicenseFilter licFilter = filters.getFilter(LicenseFilter.class); + IdentifiersFilter idFilter = filters.getFilter(IdentifiersFilter.class); + + StringBuilder builder = new StringBuilder("q=*"); + ReportUtils.enhanceLicense(builder, licFilter); + ReportUtils.enhanceDateFilter(builder, dateFilter); + ReportUtils.enhanceIdentifiers(builder, idFilter); + + String facetField = "langs"; + builder.append(String.format("&rows=0&facet=true&facet.mincount=1&facet.field=%s", facetField)); + InputStream iStream = cz.incad.kramerius.utils.solr.SolrUtils.requestWithSelectReturningStream(selectEndpint, builder.toString(), "json"); + String string = IOUtils.toString(iStream, "UTF-8"); + + ReportUtils.facetIterate(facetField, string, p-> { + Map map = new HashMap(); + map.put(COUNT_KEY, p.getValue()); + map.put(LANG_KEY, p.getKey()); + sup.processReportRecord(map); + }); + } catch (Exception e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); throw new StatisticsReportException(e); } } - @Override - public boolean verifyFilters(ReportedAction action, StatisticsFiltersContainer container) { - return true; - } + @Override + public List verifyFilters(ReportedAction action, StatisticsFiltersContainer container) { + List list = new ArrayList<>(); + DateFilter dateFilter = container.getFilter(DateFilter.class); + VerificationUtils.dateVerification(list, dateFilter.getRawFromDate()); + VerificationUtils.dateVerification(list, dateFilter.getRawToDate()); + return list; + } } diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/LicenseReport.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/LicenseReport.java new file mode 100644 index 0000000000..054a5f3b66 --- /dev/null +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/LicenseReport.java @@ -0,0 +1,122 @@ +package cz.incad.kramerius.statistics.impl; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.io.IOUtils; + +import cz.incad.kramerius.statistics.ReportedAction; +import cz.incad.kramerius.statistics.StatisticReport; +import cz.incad.kramerius.statistics.StatisticsReportException; +import cz.incad.kramerius.statistics.StatisticsReportSupport; +import cz.incad.kramerius.statistics.filters.StatisticsFiltersContainer; +import cz.incad.kramerius.statistics.utils.ReportUtils; +import cz.incad.kramerius.utils.database.Offset; + +/** + * Poskytnute licence + * @author happy + */ +public class LicenseReport extends AbstractStatisticsReport implements StatisticReport{ + + public static final Logger LOGGER = Logger.getLogger(LicenseReport.class.getName()); + + + public static final String REPORT_ID = "license"; + + @Override + public List> getReportPage(ReportedAction reportedAction, StatisticsFiltersContainer filters, + Offset rOffset) throws StatisticsReportException { + try { + String selectEndpint = super.logsEndpoint(); + + List> langs = new ArrayList<>(); + StringBuilder builder = new StringBuilder("q=*"); + applyFilters(filters, builder); + + String facetField = "provided_by_license"; + builder.append(String.format("&rows=0&facet=true&facet.mincount=1&facet.field=%s", facetField)); + InputStream iStream = cz.incad.kramerius.utils.solr.SolrUtils.requestWithSelectReturningStream(selectEndpint, builder.toString(), "json"); + String string = IOUtils.toString(iStream, "UTF-8"); + + ReportUtils.facetIterate(facetField, string, p-> { + Map map = new HashMap(); + map.put(COUNT_KEY, p.getValue()); + map.put(PROVIDED_LICENSE_KEY, p.getKey()); + langs.add(map); + }); + return langs; + } catch (IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + return new ArrayList>(); + } + } + + @Override + public List getOptionalValues(StatisticsFiltersContainer filters) { + List retvals = new ArrayList<>(); + try { + String selectEndpint = super.logsEndpoint(); + + StringBuilder builder = new StringBuilder("q=*"); + applyFilters(filters,builder); + String facetField = "provided_by_license"; + // + builder.append(String.format("&rows=0&facet=true&facet.mincount=1&facet.field=%s", facetField)); + InputStream iStream = cz.incad.kramerius.utils.solr.SolrUtils.requestWithSelectReturningStream(selectEndpint, builder.toString(), "json"); + String string = IOUtils.toString(iStream, "UTF-8"); + + ReportUtils.facetIterate(facetField, string, p-> { +// Map map = new HashMap(); +// map.put(COUNT_KEY, p.getValue()); +// map.put(PROVIDED_LICENSE_KEY, p.getKey()); + retvals.add(p.getKey().toString()); + }); + } catch (IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } + return retvals; + } + + @Override + public String getReportId() { + return REPORT_ID; + } + + + @Override + public void processAccessLog(ReportedAction action, StatisticsReportSupport sup, + StatisticsFiltersContainer filters) throws StatisticsReportException { + try { + String selectEndpint = super.logsEndpoint(); + StringBuilder builder = new StringBuilder("q=*"); + applyFilters(filters, builder); + String facetField = "provided_by_license"; + // + builder.append(String.format("&rows=0&facet=true&facet.mincount=1&facet.field=%s", facetField)); + InputStream iStream = cz.incad.kramerius.utils.solr.SolrUtils.requestWithSelectReturningStream(selectEndpint, builder.toString(), "json"); + String string = IOUtils.toString(iStream, "UTF-8"); + + ReportUtils.facetIterate(facetField, string, p-> { + Map map = new HashMap(); + map.put(COUNT_KEY, p.getValue()); + map.put(PROVIDED_LICENSE_KEY, p.getKey()); + sup.processReportRecord(map); + }); + } catch (IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } + } + + @Override + public List verifyFilters(ReportedAction action, StatisticsFiltersContainer container) { + return new ArrayList<>(); + } + +} diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/ModelStatisticReport.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/ModelStatisticReport.java index 099a2394fe..ed5b8df3b2 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/ModelStatisticReport.java +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/ModelStatisticReport.java @@ -19,6 +19,9 @@ */ package cz.incad.kramerius.statistics.impl; +import java.io.IOException; +import java.io.InputStream; +import java.net.URLEncoder; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -32,226 +35,173 @@ import cz.incad.kramerius.statistics.accesslogs.database.DatabaseStatisticsAccessLogImpl; import org.antlr.stringtemplate.StringTemplate; +import org.apache.commons.io.IOUtils; +import org.json.JSONArray; +import org.json.JSONObject; import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.name.Named; +import antlr.StringUtils; +import cz.incad.kramerius.SolrAccess; import cz.incad.kramerius.statistics.ReportedAction; import cz.incad.kramerius.statistics.StatisticReport; import cz.incad.kramerius.statistics.StatisticsReportException; import cz.incad.kramerius.statistics.StatisticsReportSupport; import cz.incad.kramerius.statistics.filters.DateFilter; import cz.incad.kramerius.statistics.filters.IPAddressFilter; +import cz.incad.kramerius.statistics.filters.IdentifiersFilter; +import cz.incad.kramerius.statistics.filters.LicenseFilter; import cz.incad.kramerius.statistics.filters.ModelFilter; import cz.incad.kramerius.statistics.filters.StatisticsFiltersContainer; import cz.incad.kramerius.statistics.filters.UniqueIPAddressesFilter; import cz.incad.kramerius.statistics.filters.VisibilityFilter; +import cz.incad.kramerius.statistics.utils.ReportUtils; import cz.incad.kramerius.utils.DatabaseUtils; +import cz.incad.kramerius.utils.conf.KConfiguration; import cz.incad.kramerius.utils.database.JDBCQueryTemplate; import cz.incad.kramerius.utils.database.JDBCUpdateTemplate; import cz.incad.kramerius.utils.database.Offset; /** + * Report pro konkretni model + * * @author pavels */ -public class ModelStatisticReport implements StatisticReport { +public class ModelStatisticReport extends AbstractStatisticsReport implements StatisticReport { public static final Logger LOGGER = Logger.getLogger(ModelStatisticReport.class.getName()); public static final String REPORT_ID = "model"; - - + @Inject - @Named("kramerius4") - Provider connectionProvider; + @Named("new-index") + SolrAccess solrAccess; + @Override public List> getReportPage(ReportedAction repAction,StatisticsFiltersContainer filters, Offset rOffset) { try { - DateFilter dateFilter = filters.getFilter(DateFilter.class); ModelFilter modelFilter = filters.getFilter(ModelFilter.class); - VisibilityFilter visFilter = filters.getFilter(VisibilityFilter.class); - UniqueIPAddressesFilter uniqueIPFilter = filters.getFilter(UniqueIPAddressesFilter.class); - - Boolean isUniqueSelected = uniqueIPFilter.getUniqueIPAddresses(); - final StringTemplate statRecord; - - if (isUniqueSelected == false) { - statRecord = DatabaseStatisticsAccessLogImpl.stGroup - .getInstanceOf("selectModelReport"); - } - else { - statRecord = DatabaseStatisticsAccessLogImpl.stGroup - .getInstanceOf("selectModelReportUnique"); - } - statRecord.setAttribute("model", modelFilter.getModel()); - statRecord.setAttribute("action", repAction != null ? repAction.name() : null); - statRecord.setAttribute("paging", true); - statRecord.setAttribute("fromDefined", dateFilter.getFromDate() != null); - statRecord.setAttribute("toDefined", dateFilter.getToDate() != null); - statRecord.setAttribute("visibility", visFilter.asMap()); + String selectEndpoint = super.logsEndpoint(); + List> models = new ArrayList<>(); + + StringBuilder builder = new StringBuilder("q=*"); + super.applyFilters(filters, builder); - @SuppressWarnings("rawtypes") - List params = StatisticUtils.jdbcParams(dateFilter, rOffset); - String sql = statRecord.toString(); - Connection conn = connectionProvider.get(); - List> models = new JDBCQueryTemplate>(conn) { - @Override - public boolean handleRow(ResultSet rs, List> returnsList) throws SQLException { - Map val = new HashMap<>(); - val.put(COUNT_KEY, rs.getInt("count")); - val.put(PID_KEY, rs.getString("pid")); - val.put(TITLE_KEY, rs.getString("title")); - val.put(MODEL_KEY, rs.getString("model")); - returnsList.add(val); - return super.handleRow(rs, returnsList); + String facetValue = "pids_"+modelFilter.getModel(); + builder.append("&rows=0&facet=true&facet.mincount=1&facet.field="+facetValue); + + InputStream iStream = cz.incad.kramerius.utils.solr.SolrUtils.requestWithSelectReturningStream(selectEndpoint, builder.toString(), "json"); + String string = IOUtils.toString(iStream, "UTF-8"); + + ReportUtils.facetIterate(facetValue, string, p-> { + Map map = new HashMap(); + map.put(COUNT_KEY, p.getValue()); + map.put(PID_KEY, p.getKey().toString()); + map.put(MODEL_KEY, modelFilter.getModel()); + try { + map.put(TITLE_KEY, titleFromSolr(p.getKey().toString())); + } catch(IOException ex) { + LOGGER.log(Level.SEVERE,ex.getMessage(),ex); } - }.executeQuery(sql, params.toArray()); - LOGGER.fine(String.format("Test statistics connection.isClosed() : %b", conn.isClosed())); + models.add(map); + }); + + + JSONObject response = new JSONObject(string).getJSONObject("response"); + Map val = new HashMap<>(); + val.put("sum", response.optInt("numFound", 0)); + models.add(val); + return models; - } catch (ParseException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - return new ArrayList>(); - } catch (SQLException ex) { - Logger.getLogger(ModelStatisticReport.class.getName()).log(Level.SEVERE, null, ex); - return new ArrayList>(); + } catch (IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + return new ArrayList>(); } + } - @Override - public List getOptionalValues() { - final StringTemplate statRecord = DatabaseStatisticsAccessLogImpl.stGroup.getInstanceOf("selectModels"); - String sql = statRecord.toString(); - Connection conn = connectionProvider.get(); - List returns = new JDBCQueryTemplate(conn) { - @Override - public boolean handleRow(ResultSet rs, List returnsList) throws SQLException { - String model = rs.getString("model"); - returnsList.add(model); - return super.handleRow(rs, returnsList); + private String titleFromSolr(Object key) throws IOException { + JSONObject result = solrAccess.requestWithSelectReturningJson(String.format("q=%s", URLEncoder.encode("pid:\""+key.toString()+"\"", "UTF-8")) ); + if (result.has("response")) { + JSONObject response = result.getJSONObject("response"); + JSONArray docs = response.getJSONArray("docs"); + if (docs.length() > 0) { + JSONObject foundDoc = docs.getJSONObject(0); + return foundDoc.optString("title.search", ""); } - }.executeQuery(sql); - try { - LOGGER.fine(String.format("Test statistics connection.isClosed() : %b", conn.isClosed())); - } catch (SQLException e) { - e.printStackTrace(); } - return returns; + return ""; } @Override - public String getReportId() { - return REPORT_ID; + public List getOptionalValues(StatisticsFiltersContainer filters) { + return new ArrayList<>(); } - - @Override - public void prepareViews(ReportedAction action, StatisticsFiltersContainer filters) throws StatisticsReportException { - try { - ModelFilter modelFilter = filters.getFilter(ModelFilter.class); - DateFilter dateFilter = filters.getFilter(DateFilter.class); - IPAddressFilter ipFilter = filters.getFilter(IPAddressFilter.class); - - - final StringTemplate statRecord = DatabaseStatisticsAccessLogImpl.stGroup.getInstanceOf("prepareModelView"); - statRecord.setAttribute("model", modelFilter.getModel()); - statRecord.setAttribute("action", action != null ? action.name() : null); - statRecord.setAttribute("paging", false); - - statRecord.setAttribute("fromDefined", dateFilter.getFromDate() != null); - statRecord.setAttribute("toDefined", dateFilter.getToDate() != null); - statRecord.setAttribute("ipaddr", ipFilter.getIpAddress()); - - String sql = statRecord.toString(); - Connection conn = connectionProvider.get(); - - String viewName = "statistics_grouped_by_sessionandpid_"+modelFilter.getModel(); - boolean tableExists = DatabaseUtils.viewExists(conn,viewName.toUpperCase()); - if (!tableExists) { - JDBCUpdateTemplate updateTemplate = new JDBCUpdateTemplate(connectionProvider.get(), true); - updateTemplate.setUseReturningKeys(false); - updateTemplate - .executeUpdate(sql); - } - // if viewExists; we have to close connection manually - if (!conn.isClosed()) { - conn.close(); - } - LOGGER.fine(String.format("Test statistics connection.isClosed() : %b", conn.isClosed())); - } catch (SQLException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - throw new StatisticsReportException(e); - } + public String getReportId() { + return REPORT_ID; } + @Override public void processAccessLog(final ReportedAction repAction, final StatisticsReportSupport sup, final StatisticsFiltersContainer filters) throws StatisticsReportException { + DateFilter dateFilter = filters.getFilter(DateFilter.class); + ModelFilter modelFilter = filters.getFilter(ModelFilter.class); + LicenseFilter licFilter = filters.getFilter(LicenseFilter.class); + IdentifiersFilter idFilter = filters.getFilter(IdentifiersFilter.class); + try { - ModelFilter modelFilter = filters.getFilter(ModelFilter.class); - DateFilter dateFilter = filters.getFilter(DateFilter.class); - VisibilityFilter visFilter = filters.getFilter(VisibilityFilter.class); - //IPAddressFilter ipAddrFilter = filters.getFilter(IPAddressFilter.class); - UniqueIPAddressesFilter uniqueIPFilter = filters.getFilter(UniqueIPAddressesFilter.class); - - Boolean isUniqueSelected = uniqueIPFilter.getUniqueIPAddresses(); - final StringTemplate statRecord; - - if (isUniqueSelected == false) { - statRecord = DatabaseStatisticsAccessLogImpl.stGroup - .getInstanceOf("selectModelReport"); - } - else { - statRecord = DatabaseStatisticsAccessLogImpl.stGroup - .getInstanceOf("selectModelReportUnique"); - } - - statRecord.setAttribute("model", modelFilter.getModel()); - statRecord.setAttribute("action", repAction != null ? repAction.name() : null); - statRecord.setAttribute("paging", false); + String selectEndpoint = super.logsEndpoint(); - statRecord.setAttribute("fromDefined", dateFilter.getFromDate() != null); - statRecord.setAttribute("toDefined", dateFilter.getToDate() != null); - statRecord.setAttribute("visibility", visFilter.asMap()); + StringBuilder builder = new StringBuilder("q=*"); + ReportUtils.enhanceLicense(builder, licFilter); + ReportUtils.enhanceDateFilter(builder, dateFilter); + ReportUtils.enhanceModelFilter(builder, modelFilter); + ReportUtils.enhanceIdentifiers(builder, idFilter); + + String facetValue = "pids_"+modelFilter.getModel(); + builder.append("&rows=0&facet=true&facet.mincount=1&facet.field="+facetValue); - @SuppressWarnings("rawtypes") - List params = StatisticUtils.jdbcParams(dateFilter); - String sql = statRecord.toString(); - Connection conn = connectionProvider.get(); + InputStream iStream = cz.incad.kramerius.utils.solr.SolrUtils.requestWithSelectReturningStream(selectEndpoint, builder.toString(), "json"); + String string = IOUtils.toString(iStream, "UTF-8"); - new JDBCQueryTemplate>(conn) { - @Override - public boolean handleRow(ResultSet rs, List> returnsList) throws SQLException { - Map val = new HashMap(); - val.put(COUNT_KEY, rs.getInt("count")); - val.put(PID_KEY, rs.getString("pid")); - val.put(TITLE_KEY, rs.getString("title")); - val.put(MODEL_KEY, rs.getString("model")); - - - sup.processReportRecord(val); - returnsList.add(val); - - return super.handleRow(rs, returnsList); + ReportUtils.facetIterate(facetValue, string, p-> { + Map map = new HashMap(); + map.put(COUNT_KEY, p.getValue()); + map.put(PID_KEY, p.getKey().toString()); + map.put(MODEL_KEY, modelFilter.getModel()); + try { + map.put(TITLE_KEY, titleFromSolr(p.getKey().toString())); + } catch(IOException ex) { + LOGGER.log(Level.SEVERE,ex.getMessage(),ex); } - }.executeQuery(sql,params.toArray()); - LOGGER.fine(String.format("Test statistics connection.isClosed() : %b", conn.isClosed())); - } catch (ParseException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - throw new StatisticsReportException(e); - } catch (SQLException ex) { - Logger.getLogger(ModelStatisticReport.class.getName()).log(Level.SEVERE, null, ex); + sup.processReportRecord(map); + }); + } catch (IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); } } @Override - public boolean verifyFilters(ReportedAction action, StatisticsFiltersContainer container) { - ModelFilter modelFilter = container.getFilter(ModelFilter.class); - return modelFilter.getModel() != null; - } + public List verifyFilters(ReportedAction action, StatisticsFiltersContainer container) { + List list = new ArrayList<>(); + ModelFilter modelFilter = container.getFilter(ModelFilter.class); + if (modelFilter.getModel() == null) { + list.add("model is mandatory"); + } + DateFilter dateFilter = container.getFilter(DateFilter.class); + VerificationUtils.dateVerification(list, dateFilter.getRawFromDate()); + VerificationUtils.dateVerification(list, dateFilter.getRawToDate()); + + return list; + + } } diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/ModelSummaryReport.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/ModelSummaryReport.java new file mode 100644 index 0000000000..9221bd1088 --- /dev/null +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/ModelSummaryReport.java @@ -0,0 +1,166 @@ +package cz.incad.kramerius.statistics.impl; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import org.antlr.stringtemplate.StringTemplate; + +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.google.inject.name.Named; + +import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.statistics.ReportedAction; +import cz.incad.kramerius.statistics.StatisticReport; +import cz.incad.kramerius.statistics.StatisticsReportException; +import cz.incad.kramerius.statistics.StatisticsReportSupport; +import cz.incad.kramerius.statistics.accesslogs.database.DatabaseStatisticsAccessLogImpl; +import cz.incad.kramerius.statistics.filters.DateFilter; +import cz.incad.kramerius.statistics.filters.IdentifiersFilter; +import cz.incad.kramerius.statistics.filters.LicenseFilter; +import cz.incad.kramerius.statistics.filters.ModelFilter; +import cz.incad.kramerius.statistics.filters.MultimodelFilter; +import cz.incad.kramerius.statistics.filters.StatisticsFilter; +import cz.incad.kramerius.statistics.filters.StatisticsFiltersContainer; +import cz.incad.kramerius.statistics.filters.VisibilityFilter; +import cz.incad.kramerius.utils.database.JDBCQueryTemplate; +import cz.incad.kramerius.utils.database.Offset; + +/** + * Agregovany report -> Modely a pocty + * @author happy + * + */ +public class ModelSummaryReport implements StatisticReport { + + private static final List SUPPORTED_MODELS = Arrays.asList("monograph", "periodical", "article", "convolute", "map", "graphic", "archive", + "manuscript", "soundrecording", "collection"); + + public static final Logger LOGGER = Logger.getLogger(ModelSummaryReport.class.getName()); + + public static final String REPORT_ID = "summary"; + + @Inject + @Named("new-index") + SolrAccess solrAccess; + + + @Override + public List> getReportPage(ReportedAction reportedAction, StatisticsFiltersContainer filters, + Offset rOffset) throws StatisticsReportException { + + MultimodelFilter mfilter = new MultimodelFilter(); + mfilter.setModels(SUPPORTED_MODELS); + + Map multimodels = new HashMap<>(); + Map sums = new HashMap<>(); + + mfilter.getModels().stream().forEach(model -> { + ModelFilter modelFilter = new ModelFilter(); + modelFilter.setModel(model); + + DateFilter dateFilter = filters.getFilter(DateFilter.class); + VisibilityFilter visFilter = filters.getFilter(VisibilityFilter.class); + LicenseFilter licFilter = filters.getFilter(LicenseFilter.class); + IdentifiersFilter idFilter = filters.getFilter(IdentifiersFilter.class); + + StatisticsFiltersContainer container = new StatisticsFiltersContainer( + new StatisticsFilter[] { dateFilter, visFilter, licFilter, modelFilter,idFilter }); + + ModelStatisticReport report = new ModelStatisticReport(); + report.solrAccess = this.solrAccess; + List> reportPage = report.getReportPage(reportedAction, container, rOffset); + reportPage.stream().forEach(it -> { + if (it.containsKey("sum")) { + Integer sum = (Integer) it.get("sum"); + if (sum > 0) { + sums.put(model, (Integer) it.get("sum")); + } + it.remove("sum"); + } + }); + }); + + multimodels.put("sums", sums); + return Arrays.asList(multimodels); + } + + @Override + public List getOptionalValues(StatisticsFiltersContainer filters) { + return new ArrayList<>(); + } + + @Override + public String getReportId() { + return REPORT_ID; + } + + + @Override + public void processAccessLog(ReportedAction action, StatisticsReportSupport sup, StatisticsFiltersContainer filters) + throws StatisticsReportException { + + + MultimodelFilter mfilter = new MultimodelFilter(); + mfilter.setModels(SUPPORTED_MODELS); + +// Map multimodels = new HashMap<>(); +// Map sums = new HashMap<>(); + + mfilter.getModels().stream().forEach(model -> { + ModelFilter modelFilter = new ModelFilter(); + modelFilter.setModel(model); + + DateFilter dateFilter = filters.getFilter(DateFilter.class); + VisibilityFilter visFilter = filters.getFilter(VisibilityFilter.class); + LicenseFilter licFilter = filters.getFilter(LicenseFilter.class); + IdentifiersFilter idFilter = filters.getFilter(IdentifiersFilter.class); + + StatisticsFiltersContainer container = new StatisticsFiltersContainer( + new StatisticsFilter[] { dateFilter, visFilter, licFilter, modelFilter,idFilter }); + + ModelStatisticReport report = new ModelStatisticReport(); + report.solrAccess = this.solrAccess; + + List> reportPage = report.getReportPage(action, container, null); + reportPage.stream().forEach(it -> { + if (it.containsKey("sum")) { + Integer sum = (Integer) it.get("sum"); + if (sum > 0) { + Map map = new HashMap(); + map.put(COUNT_KEY, (Integer) it.get("sum")); + map.put(MODEL_KEY, model); + sup.processReportRecord(map); + } + } + }); + }); + } + + + + @Override + public boolean convertToObject() { + return true; + } + + @Override + public List verifyFilters(ReportedAction action, StatisticsFiltersContainer container) { + List list = new ArrayList<>(); + DateFilter dateFilter = container.getFilter(DateFilter.class); + VerificationUtils.dateVerification(list, dateFilter.getRawFromDate()); + VerificationUtils.dateVerification(list, dateFilter.getRawToDate()); + return list; + } + +} diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/MultimodelReport.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/MultimodelReport.java new file mode 100644 index 0000000000..e16f41f5ee --- /dev/null +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/MultimodelReport.java @@ -0,0 +1,211 @@ +package cz.incad.kramerius.statistics.impl; + +import java.io.IOException; +import java.io.InputStream; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import org.antlr.stringtemplate.StringTemplate; +import org.apache.commons.io.IOUtils; +import org.json.JSONArray; +import org.json.JSONObject; + +import com.google.inject.Inject; +import com.google.inject.Provider; +import com.google.inject.name.Named; + +import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.statistics.ReportedAction; +import cz.incad.kramerius.statistics.StatisticReport; +import cz.incad.kramerius.statistics.StatisticsReportException; +import cz.incad.kramerius.statistics.StatisticsReportSupport; +import cz.incad.kramerius.statistics.accesslogs.database.DatabaseStatisticsAccessLogImpl; +import cz.incad.kramerius.statistics.filters.DateFilter; +import cz.incad.kramerius.statistics.filters.IdentifiersFilter; +import cz.incad.kramerius.statistics.filters.LicenseFilter; +import cz.incad.kramerius.statistics.filters.ModelFilter; +import cz.incad.kramerius.statistics.filters.MultimodelFilter; +import cz.incad.kramerius.statistics.filters.StatisticsFilter; +import cz.incad.kramerius.statistics.filters.StatisticsFiltersContainer; +import cz.incad.kramerius.statistics.filters.VisibilityFilter; +import cz.incad.kramerius.utils.conf.KConfiguration; +import cz.incad.kramerius.utils.database.JDBCQueryTemplate; +import cz.incad.kramerius.utils.database.Offset; + +/** + * Agregovany report -> Modely a nejcasteji poskytaovana dila dle modelu + * @author happy + * + */ +public class MultimodelReport implements StatisticReport { + private static final List SUPPORTED_MODELS = Arrays.asList("monograph", "periodical", "article", "convolute", "map", "graphic", "archive", + "manuscript", "soundrecording", "collection"); + + public static final Logger LOGGER = Logger.getLogger(MultimodelReport.class.getName()); + + public static final String REPORT_ID = "multimodel"; + + @Inject + @Named("new-index") + SolrAccess solrAccess; + + @Override + public List> getReportPage(ReportedAction reportedAction, StatisticsFiltersContainer filters, + Offset rOffset) throws StatisticsReportException { + + MultimodelFilter mfilter = new MultimodelFilter(); + mfilter.setModels(SUPPORTED_MODELS); + + Map multimodels = new HashMap<>(); + Map sums = new HashMap<>(); + + mfilter.getModels().stream().forEach(model -> { + ModelFilter modelFilter = new ModelFilter(); + modelFilter.setModel(model); + + DateFilter dateFilter = filters.getFilter(DateFilter.class); + VisibilityFilter visFilter = filters.getFilter(VisibilityFilter.class); + LicenseFilter licFilter = filters.getFilter(LicenseFilter.class); + IdentifiersFilter idFilter = filters.getFilter(IdentifiersFilter.class); + + StatisticsFiltersContainer container = new StatisticsFiltersContainer( + new StatisticsFilter[] { dateFilter, visFilter, licFilter, modelFilter, idFilter }); + + ModelStatisticReport report = new ModelStatisticReport(); + report.solrAccess = this.solrAccess; + + List> reportPage = report.getReportPage(reportedAction, container, rOffset); + reportPage.stream().forEach(it -> { + if (it.containsKey("sum")) { + Integer sum = (Integer) it.get("sum"); + if (sum > 0) { + sums.put(model, (Integer) it.get("sum")); + } + it.remove("sum"); + } + }); + + reportPage = reportPage.stream().filter(it -> { + return !it.isEmpty(); + }).collect(Collectors.toList()); + + if (!reportPage.isEmpty()) { + multimodels.put(model, reportPage); + } + }); + multimodels.put("sums", sums); + return Arrays.asList(multimodels); + } + + @Override + public List getOptionalValues(StatisticsFiltersContainer filters) { + return new ArrayList<>(); + } + + @Override + public String getReportId() { + return REPORT_ID; + } + + + + @Override + public boolean convertToObject() { + return true; + } + + @Override + public void processAccessLog(ReportedAction action, StatisticsReportSupport sup, StatisticsFiltersContainer filters) + throws StatisticsReportException { +// try { +// ModelFilter modelFilter = filters.getFilter(ModelFilter.class); +// DateFilter dateFilter = filters.getFilter(DateFilter.class); +// VisibilityFilter visFilter = filters.getFilter(VisibilityFilter.class); +// LicenseFilter licFilter = filters.getFilter(LicenseFilter.class); +// +// final StringTemplate statRecord = DatabaseStatisticsAccessLogImpl.stGroup +// .getInstanceOf("selectModelReport"); +// +// final StringTemplate counts = DatabaseStatisticsAccessLogImpl.stGroup +// .getInstanceOf("selectModelReportCounts"); +// +// statRecord.setAttribute("model", modelFilter.getModel()); +// statRecord.setAttribute("action", action != null ? action.name() : null); +// statRecord.setAttribute("paging", false); +// +// statRecord.setAttribute("fromDefined", dateFilter.getFromDate() != null); +// statRecord.setAttribute("toDefined", dateFilter.getToDate() != null); +// statRecord.setAttribute("visibility", visFilter.asMap()); +// +// statRecord.setAttribute("licenseDefined", licFilter.getLicence() != null); +// //statRecord.setAttribute("license", licFilter.getLicence()); +// +// counts.setAttribute("model", modelFilter.getModel()); +// counts.setAttribute("action", action != null ? action.name() : null); +// counts.setAttribute("paging", false); +// counts.setAttribute("fromDefined", dateFilter.getFromDate() != null); +// counts.setAttribute("toDefined", dateFilter.getToDate() != null); +// counts.setAttribute("visibility", visFilter.asMap()); +// counts.setAttribute("licenseDefined", licFilter.getLicence() != null); +// +// @SuppressWarnings("rawtypes") +// List params = StatisticUtils.jdbcParams(dateFilter); +// String sql = statRecord.toString(); +// Connection conn = connectionProvider.get(); +// +// new JDBCQueryTemplate>(conn) { +// @Override +// public boolean handleRow(ResultSet rs, List> returnsList) throws SQLException { +// Map val = new HashMap(); +// val.put(COUNT_KEY, rs.getInt("count")); +// val.put(PID_KEY, rs.getString("pid")); +// val.put(TITLE_KEY, rs.getString("title")); +// val.put(MODEL_KEY, rs.getString("model")); +// +// +// sup.processReportRecord(val); +// returnsList.add(val); +// +// return super.handleRow(rs, returnsList); +// } +// }.executeQuery(sql,params.toArray()); +// +// List> sum = new JDBCQueryTemplate>(connectionProvider.get()) { +// @Override +// public boolean handleRow(ResultSet rs, List> returnsList) throws SQLException { +// Map val = new HashMap<>(); +// val.put(modelFilter.getModel(), rs.getInt("sum")); +// returnsList.add(val); +// return super.handleRow(rs, returnsList); +// } +// }.executeQuery(counts.toString(), StatisticUtils.jdbcParams(dateFilter, licFilter,null).toArray()); +// +// LOGGER.fine(String.format("Test statistics connection.isClosed() : %b", conn.isClosed())); +// } catch (ParseException e) { +// LOGGER.log(Level.SEVERE, e.getMessage(), e); +// throw new StatisticsReportException(e); +// } catch (SQLException ex) { +// Logger.getLogger(ModelStatisticReport.class.getName()).log(Level.SEVERE, null, ex); +// } + } + + @Override + public List verifyFilters(ReportedAction action, StatisticsFiltersContainer container) { + List list = new ArrayList<>(); + DateFilter dateFilter = container.getFilter(DateFilter.class); + VerificationUtils.dateVerification(list, dateFilter.getRawFromDate()); + VerificationUtils.dateVerification(list, dateFilter.getRawToDate()); + return list; + } + +} diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/NKPLogReport.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/NKPLogReport.java deleted file mode 100644 index 9f45a6da18..0000000000 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/NKPLogReport.java +++ /dev/null @@ -1,592 +0,0 @@ -package cz.incad.kramerius.statistics.impl; - -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.google.inject.name.Named; -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.ObjectPidsPath; -import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.database.VersionService; -import cz.incad.kramerius.statistics.ReportedAction; -import cz.incad.kramerius.statistics.StatisticReport; -import cz.incad.kramerius.statistics.StatisticsReportException; -import cz.incad.kramerius.statistics.StatisticsReportSupport; -import cz.incad.kramerius.statistics.accesslogs.dnnt.DNNTStatisticsAccessLogImpl; -import cz.incad.kramerius.statistics.accesslogs.dnnt.date.YearLogFormat; -import cz.incad.kramerius.statistics.accesslogs.utils.SElemUtils; -import cz.incad.kramerius.statistics.filters.DateFilter; -import cz.incad.kramerius.statistics.filters.StatisticsFiltersContainer; -import cz.incad.kramerius.statistics.accesslogs.database.DatabaseStatisticsAccessLogImpl; -import cz.incad.kramerius.statistics.filters.VisibilityFilter; -import cz.incad.kramerius.utils.conf.KConfiguration; -import cz.incad.kramerius.utils.database.JDBCQueryTemplate; -import cz.incad.kramerius.utils.database.Offset; -import cz.incad.kramerius.utils.solr.SolrUtils; -import org.antlr.stringtemplate.StringTemplate; -import org.json.JSONArray; -import org.json.JSONObject; -import org.w3c.dom.Document; - -import java.io.IOException; -import java.sql.*; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.*; -import java.util.function.Consumer; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import static cz.incad.kramerius.database.cond.ConditionsInterpretHelper.versionCondition; - - -/** - * Creates report for NKP - * - */ -public class NKPLogReport implements StatisticReport { - - public static final String RUNTIME_ATTRS = "runtimeAttributes"; - public static final String MISSING_ATTRS = "missingAttributes"; - //public static List EXPECTED_FIELDS = Arrays.asList(""); - - public static final SimpleDateFormat ACCESS_DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); - - public static final Logger LOGGER = Logger.getLogger(AuthorReport.class.getName()); - - public static final String REPORT_ID = "nkp"; - - - @Inject - @Named("kramerius4") - Provider connectionProvider; - - @Inject - @Named("new-index") - SolrAccess solrAccess; - - @Inject - @Named("cachedFedoraAccess") - FedoraAccess fedoraAccess; - - @Inject - VersionService versionService; - - - @Override - public List> getReportPage(ReportedAction reportedAction, StatisticsFiltersContainer filters, Offset rOffset) throws StatisticsReportException { - throw new UnsupportedOperationException("unsupported for this report"); - } - - @Override - public List getOptionalValues() { - return null; - } - - @Override - public String getReportId() { - return REPORT_ID; - } - - @Override - public void prepareViews(ReportedAction action, StatisticsFiltersContainer container) throws StatisticsReportException { } - - @Override - public void processAccessLog(ReportedAction action, StatisticsReportSupport sup, StatisticsFiltersContainer filters) throws StatisticsReportException { - try { - - DateFilter dateFilter = filters.getFilter(DateFilter.class); - VisibilityFilter visFilter = filters.getFilter(VisibilityFilter.class); - - if (dateFilter.getFromDate() != null && dateFilter.getToDate() != null) { - final StringTemplate statRecord = DatabaseStatisticsAccessLogImpl.stGroup.getInstanceOf("nkpLogsReport"); - statRecord.setAttribute("action", ReportedAction.READ.name()); - - statRecord.setAttribute("fromDefined", dateFilter.getFromDate() != null); - statRecord.setAttribute("toDefined", dateFilter.getToDate() != null); - // visibility must be filtered in the runtime - //statRecord.setAttribute("visibility", visFilter.asMap()); - - - @SuppressWarnings("rawtypes") - List params = StatisticUtils.jdbcParams(dateFilter); - String sql = statRecord.toString(); - Connection conn = connectionProvider.get(); - - new StastisticsIteration(sql, params, conn, collectedRecord-> { - logReport(collectedRecord, sup, visFilter); - }).iterate(); - - } else { - throw new UnsupportedOperationException("Full report is not supported. Please, use dateFrom and dateTo"); - } - - } catch (ParseException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } - - } - - static class StastisticsIteration extends JDBCQueryTemplate { - - - private Record currentRecord = new Record(); - - private Consumer consumer; - private String sql; - private List params; - - public StastisticsIteration(String sql, List params, Connection connection, Consumer consumer) { - super(connection); - this.consumer = consumer; - this.params = params; - this.sql = sql; - } - - public void iterate() { - executeQuery(sql,params.toArray()); - if (currentRecord != null && currentRecord.pid != null) consumer.accept(currentRecord); - } - - @Override - public boolean handleRow(ResultSet rs, List returnsList) throws SQLException { - try { - int recordId = rs.getInt("slrecord_id"); - int detailId = rs.getInt("sddetail_id"); - int authorId = rs.getInt("saauthor_id"); - int publisherId = rs.getInt("sppublisher_id"); - - if (currentRecord.isDifferent(recordId)) { - if (currentRecord.recordId != -1) { - consumer.accept(currentRecord); - } - currentRecord = Record.load(rs); - } - - if (currentRecord.lastDetail() == null || currentRecord.lastDetail().isDifferent(detailId)) { - currentRecord.details.add(Detail.load(rs)); - } - - if (authorId != -1) { - currentRecord.lastDetail().authors.add(Author.load(rs)); - } - if (publisherId != -1) { - currentRecord.lastDetail().publishers.add(Publisher.load(rs)); - } - - - - - } catch (Exception e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } - return super.handleRow(rs, returnsList); - } - - } - - - - void logReport(Record record, StatisticsReportSupport sup, VisibilityFilter visibilityFilter) { - - // Must be filtered in runtime - boolean selected = visibilityFilter.getSelected().equals(VisibilityFilter.VisbilityType.ALL); - if (!selected) { - - List firstBranch = record.details.stream().filter(detail -> { - return detail.branchId == 0; - }).collect(Collectors.toList()); - - Detail detail = firstBranch.get(0); - if (detail != null && detail.rights != null) { - String trim = detail.rights.trim(); - selected = trim.contains(visibilityFilter.getSelected().name().toLowerCase()); - } - } - - - if (selected) { - // disable solr and fedora access - // disable solr and fedora access - boolean disbleFedoraAccess = KConfiguration.getInstance().getConfiguration().getBoolean("nkp.logs.disablefedoraaccess", false); - Map map = record.toMap(); - Object dbversion = map.get("dbversion"); - if (dbversion != null) map.remove("dbversion"); - if (dbversion == null || versionCondition(dbversion.toString(), "<", "6.6.6")) { - try { - if (!disbleFedoraAccess) { - // only one place where we are connecting - Document solrDoc = solrAccess.getSolrDataByPid(record.pid); - if (solrDoc != null) { - - map.put(DNNTStatisticsAccessLogImpl.SOLR_DATE_KEY, new YearLogFormat().format(SElemUtils.selem("str", "datum_str", solrDoc))); - // fill publishers - ObjectPidsPath[] paths = solrAccess.getPidPaths(null, solrDoc); - List dcPublishers = DNNTStatisticsAccessLogImpl.dcPublishers(paths, fedoraAccess); - if (!dcPublishers.isEmpty()) { - JSONArray publishersArray = new JSONArray(); - for (int i=0,ll=dcPublishers.size();i> identifiers = DNNTStatisticsAccessLogImpl.identifiers(paths, fedoraAccess); - identifiers.keySet().forEach(key-> map.put(key, identifiers.get(key))); - - List dnntLabels = SolrUtils.disectLicenses(solrDoc.getDocumentElement()); - map.put(DNNTStatisticsAccessLogImpl.DNNT_LABELS_KEY, dnntLabels); - - String s = SolrUtils.disectDNNTFlag(solrDoc.getDocumentElement()); - if (s != null) { - map.put(DNNTStatisticsAccessLogImpl.DNNT_KEY, Boolean.valueOf(s)); - } - - List runtimeFileds = new ArrayList<>(Arrays.asList(DNNTStatisticsAccessLogImpl.SOLR_DATE_KEY , - DNNTStatisticsAccessLogImpl.PUBLISHERS_KEY, - DNNTStatisticsAccessLogImpl.DNNT_KEY, - DNNTStatisticsAccessLogImpl.DNNT_LABELS_KEY, - DNNTStatisticsAccessLogImpl.PROVIDED_BY_DNNT_KEY)); - identifiers.keySet().forEach(runtimeFileds::add); - - map.put(RUNTIME_ATTRS, runtimeFileds); - map.put(MISSING_ATTRS, Arrays.asList("shibboleth", "providedByLabel")); - } - } else { - map.put(MISSING_ATTRS, Arrays.asList("shibboleth", "providedByLabel", - DNNTStatisticsAccessLogImpl.SOLR_DATE_KEY , - DNNTStatisticsAccessLogImpl.PUBLISHERS_KEY, - DNNTStatisticsAccessLogImpl.DNNT_KEY, - DNNTStatisticsAccessLogImpl.DNNT_LABELS_KEY, - DNNTStatisticsAccessLogImpl.PROVIDED_BY_DNNT_KEY - )); - - } - } catch (IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } - } else if (dbversion != null && versionCondition(dbversion.toString(), "=", "6.6.6")) { - if (!disbleFedoraAccess) { - try { - Document solrDoc = solrAccess.getSolrDataByPid(record.pid); - if (solrDoc != null) { - List dnntLabels = SolrUtils.disectLicenses(solrDoc.getDocumentElement()); - map.put(DNNTStatisticsAccessLogImpl.DNNT_LABELS_KEY, dnntLabels); - map.put(RUNTIME_ATTRS, Arrays.asList(DNNTStatisticsAccessLogImpl.DNNT_LABELS_KEY)); - } - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } - } else { - map.put(MISSING_ATTRS, Arrays.asList(DNNTStatisticsAccessLogImpl.DNNT_LABELS_KEY)); - } - } - sup.processReportRecord(map); - } - } - - @Override - public boolean verifyFilters(ReportedAction action, StatisticsFiltersContainer filters) { - DateFilter dateFilter = filters.getFilter(DateFilter.class); - return dateFilter.getToDate() != null && dateFilter.getFromDate() != null; - } - - - static class Record { - - private int recordId=-1; - private String pid; - private String date; - private String remoteIpAddress; - private String user; - private String requestedUrl; - private Boolean dnnt; - private Boolean providedbydnnt; - private String evaluateMap; - private String userSessionAttributes; - private String dbversion; - - private List dnntLabels = new ArrayList<>(); - - - private List details = new ArrayList<>(); - - Map toMap() { - Map map = new HashMap(); - map.put("pid", this.pid); - map.put("date", this.date); - map.put("remoteAddr", this.remoteIpAddress); - if (this.user != null) map.put("username", this.user); - if (this.dnnt != null) map.put("dnnt",this.dnnt); - if (this.providedbydnnt != null) map.put("providedByDnnt", this.providedbydnnt); - if (this.evaluateMap != null) { - JSONObject evalmap = new JSONObject(this.evaluateMap); - for (Object key : evalmap.keySet()) { - map.put(key.toString(), evalmap.get(key.toString())); - } - } - if (this.userSessionAttributes != null) { - JSONObject uSessionMap = new JSONObject(userSessionAttributes); - for (Object key : uSessionMap.keySet()) { - map.put(key.toString(), uSessionMap.get(key.toString())); - } - } - - map.put("dbversion", this.dbversion); - - if (!this.details.isEmpty()) { - // filter branch 0 - List firstBranch = this.details.stream().filter(detail -> { - return detail.branchId == 0; - }).collect(Collectors.toList()); - - List secondBranch = this.details.stream().filter(detail -> { - return detail.branchId == 1; - }).collect(Collectors.toList()); - - map.put("rootTitle", firstBranch.get(firstBranch.size() -1).title); - map.put("dcTitle", firstBranch.get(0).title); - - map.put("rootPid", firstBranch.get(firstBranch.size() -1).pid); - map.put("rootModel", firstBranch.get(firstBranch.size() -1).model); - - map.put("policy", firstBranch.get(0).rights); - - List pidPaths = new ArrayList<>(); - List modelPaths = new ArrayList<>(); - - List firstBranchPids = firstBranch.stream().map(detail -> { - return detail.pid; - }).collect(Collectors.toList()); - Collections.reverse(firstBranchPids); - pidPaths.add(firstBranchPids.stream().collect(Collectors.joining("/"))); - - if(!secondBranch.isEmpty()) { - List secondBranchPids = secondBranch.stream().map(detail -> { - return detail.pid; - }).collect(Collectors.toList()); - Collections.reverse(secondBranchPids); - pidPaths.add(secondBranchPids.stream().collect(Collectors.joining("/"))); - } - - List firstBranchModels = firstBranch.stream().map(detail -> { - return detail.model; - }).collect(Collectors.toList()); - Collections.reverse(firstBranchModels); - modelPaths.add(firstBranchModels.stream().collect(Collectors.joining("/"))); - - if(!secondBranch.isEmpty()) { - List secondBranchModels = secondBranch.stream().map(detail -> { - return detail.model; - }).collect(Collectors.toList()); - Collections.reverse(secondBranchModels); - modelPaths.add(secondBranchModels.stream().collect(Collectors.joining("/"))); - } - - - //map.put("models_path", ) - map.put("models_path", modelPaths); - map.put("pids_path", pidPaths); - - Optional issuedDate = details.stream().map(detail -> { - return detail.issuedDate; - }).filter(Objects::nonNull).findFirst(); - - if (issuedDate.isPresent()) { - map.put("publishedDate", issuedDate.get()); - } - - Optional solrDate = details.stream().map(detail -> { - return detail.solrDate; - }).filter(Objects::nonNull).findFirst(); - - if (solrDate.isPresent()) { - map.put("solrDate", solrDate.get()); - } - - List publishers = details.stream().map(detail -> { - return detail.publishers; - }).flatMap(Collection::stream).map(publisher -> { - return publisher.publisher; - }).filter(Objects::nonNull).distinct().collect(Collectors.toList()); - - if (!publishers.isEmpty()) { - map.put("publishers", publishers); - } - - List authors = details.stream().map(detail -> { - return detail.authors; - }).flatMap(Collection::stream).map(author -> { - return author.author; - }).filter(Objects::nonNull).distinct().collect(Collectors.toList()); - - if (!authors.isEmpty()) { - map.put("authors", authors); - } - - List isbn = details.stream().map(detail -> { - return detail.isbn; - }).flatMap(Collection::stream).filter(Objects::nonNull).distinct().collect(Collectors.toList()); - if (!isbn.isEmpty()) map.put("isbn", isbn); - - List issn = details.stream().map(detail -> { - return detail.issn; - }).flatMap(Collection::stream).filter(Objects::nonNull).distinct().collect(Collectors.toList()); - if (!issn.isEmpty()) map.put("issn", issn); - - List ccnb = details.stream().map(detail -> { - return detail.ccnb; - }).flatMap(Collection::stream).filter(Objects::nonNull).distinct().collect(Collectors.toList()); - if (!ccnb.isEmpty()) map.put("ccnb", ccnb); - - } - return map; - } - - - - private static Record load(ResultSet rs) throws SQLException { - Record nrecord = new Record(); - nrecord.recordId= rs.getInt("slrecord_id"); - nrecord.pid = rs.getString("slpid"); - nrecord.date = ACCESS_DATE_FORMAT.format(rs.getTimestamp("sldate")); - nrecord.remoteIpAddress=rs.getString("slremote_ip_address"); - nrecord.user=rs.getString("slUSER"); - - // test on 9.2 - nrecord.dnnt= (Boolean) rs.getObject("sldnnt");// rs.getBoolean("sldnnt"); - // if null; must be provided by dnnt false - nrecord.providedbydnnt= rs.getBoolean("slprovidedByDnnt"); - - nrecord.evaluateMap=rs.getString("slevaluatemap"); - nrecord.userSessionAttributes=rs.getString("slusersessionattributes"); - nrecord.dbversion = rs.getString("sldbversion"); - - // sldnnt_labels - Array dnntLabels = rs.getArray("sldnnt_labels"); - if (dnntLabels != null) { - String[] sdissnArray = (String[]) dnntLabels.getArray(); - if (sdissnArray != null) nrecord.dnntLabels.addAll(Arrays.stream(sdissnArray).collect(Collectors.toList())); - } - - return nrecord; - } - - public boolean isDifferent(int rId) { - return this.recordId != rId; - } - - - public Detail lastDetail() { - return details.isEmpty() ? null : details.get(details.size() -1); - } - } - - static class Detail { - - int detailId=-1; - String pid; - String model; - String issuedDate; - String solrDate; - String rights; - String title; - int branchId; - - List issn = new ArrayList<>(); - List isbn = new ArrayList<>(); - List ccnb = new ArrayList<>(); - - private List authors = new ArrayList<>(); - private List publishers = new ArrayList<>(); - - private static Detail load(ResultSet rs) throws SQLException { - Detail detail = new Detail(); - detail.detailId = rs.getInt("sddetail_id"); - detail.pid = rs.getString("sdpid"); - detail.model = rs.getString("sdmodel"); - detail.issuedDate = new YearLogFormat().format(rs.getString("sdissued_date")); - detail.solrDate = new YearLogFormat().format(rs.getString("sdsolr_date")); - String rights = rs.getString("sdrights"); - detail.rights= rights != null ? rights.contains(":") ? rights.split(":")[1] : rights : null; - detail.title= rs.getString("sdtitle"); - detail.branchId = rs.getInt("sdbranch_id"); - - Array sdissn = rs.getArray("sdissn"); - if (sdissn != null) { - String[] sdissnArray = (String[]) sdissn.getArray(); - if (sdissnArray != null) detail.issn.addAll(Arrays.stream(sdissnArray).collect(Collectors.toList())); - } - - Array sdisbn = rs.getArray("sdisbn"); - if (sdisbn != null) { - String[] sdisbnArray = (String[]) sdisbn.getArray(); - if (sdisbnArray != null) detail.isbn.addAll(Arrays.stream(sdisbnArray).collect(Collectors.toList())); - } - - Array sdccnb = rs.getArray("sdccnb"); - if (sdccnb != null) { - String[] sdccnbArray = (String[]) sdccnb.getArray(); - if (sdccnbArray != null) detail.ccnb.addAll(Arrays.stream(sdccnbArray).collect(Collectors.toList())); - } - - return detail; - } - - public Author lastAuthor() { - return authors.isEmpty() ? null : authors.get(authors.size() -1); - } - - public Publisher lastPublisher() { - return publishers.isEmpty() ? null : publishers.get(publishers.size() -1); - } - - public boolean isDifferent(int dId) { - return this.detailId != dId; - } - } - - static class Author { - - int authorId=-1; - String author; - - private static Author load(ResultSet rs) throws SQLException { - int authorId = rs.getInt("saauthor_id"); - if (authorId != -1) { - Author author = new Author(); - author.authorId = rs.getInt("saauthor_id"); - author.author = rs.getString("saauthor_name"); - return author; - } else return null; - } - - public boolean isDifferent(int aId) { - return authorId != aId; - } - - } - - static class Publisher { - - int publisherId=-1; - String publisher; - - - private static Publisher load(ResultSet rs) throws SQLException { - int publisherId = rs.getInt("sppublisher_id"); - if (publisherId != -1) { - Publisher publisher = new Publisher(); - publisher.publisherId = rs.getInt("sppublisher_id"); - publisher.publisher = rs.getString("sppublisher_name"); - return publisher; - } else return null; - } - - public boolean isDifferent(int pId) { - return publisherId != pId; - } - - } -} diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/PidsReport.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/PidsReport.java index 518b0d539c..918c1ac9eb 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/PidsReport.java +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/PidsReport.java @@ -33,7 +33,7 @@ import cz.incad.kramerius.utils.database.Offset; import java.util.logging.Logger; -public class PidsReport implements StatisticReport { +public class PidsReport extends AbstractStatisticsReport implements StatisticReport { static java.util.logging.Logger LOGGER = java.util.logging.Logger.getLogger(PidsReport.class.getName()); @@ -73,7 +73,7 @@ public List> getReportPage(ReportedAction repAction, Statist statRecord.setAttribute("ipaddr", ipFilter.getIpAddress()); @SuppressWarnings("rawtypes") - List params = StatisticUtils.jdbcParams(dateFilter, rOffset); + List params = StatisticUtils.jdbcParams(dateFilter,null, rOffset); String sql = statRecord.toString(); Connection conn = connectionProvider.get(); @@ -99,7 +99,7 @@ public boolean handleRow(ResultSet rs, List> returnsList) th } @Override - public List getOptionalValues() { + public List getOptionalValues(StatisticsFiltersContainer filters) { return new ArrayList(); } @@ -109,40 +109,6 @@ public String getReportId() { } - @Override - public void prepareViews(ReportedAction action, StatisticsFiltersContainer filters) throws StatisticsReportException { - try { - IPAddressFilter ipFilter = filters.getFilter(IPAddressFilter.class); - PidsFilter pidsFilter = filters.getFilter(PidsFilter.class); - - final StringTemplate statRecord = DatabaseStatisticsAccessLogImpl.stGroup.getInstanceOf("preparePidsView"); - statRecord.setAttribute("action", action != null ? action.name() : null); - statRecord.setAttribute("paging", false); - - statRecord.setAttribute("pids", pidsFilter.getPids().split(",")); - statRecord.setAttribute("ipaddr", ipFilter.getIpAddress()); - - String sql = statRecord.toString(); - Connection conn = connectionProvider.get(); - - String viewName = "statistics_grouped_by_sessionandpid_pid"; - boolean tableExists = DatabaseUtils.viewExists(conn, viewName.toUpperCase()); - if (!tableExists) { - JDBCUpdateTemplate updateTemplate = new JDBCUpdateTemplate(conn, true); - updateTemplate.setUseReturningKeys(false); - updateTemplate - .executeUpdate(sql); - } - // if table exists; we have to close connection manually - if (!conn.isClosed()) { - conn.close(); - } - LOGGER.fine(String.format("Test statistics connection.isClosed() : %b", conn.isClosed())); - } catch (SQLException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - throw new StatisticsReportException(e); - } - } @Override @@ -198,8 +164,8 @@ public boolean handleRow(ResultSet rs, List> returnsList) th } } - @Override - public boolean verifyFilters(ReportedAction action, StatisticsFiltersContainer container) { - return true; - } + @Override + public List verifyFilters(ReportedAction action, StatisticsFiltersContainer container) { + return new ArrayList<>(); + } } diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/StatisticUtils.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/StatisticUtils.java index 7b1a579309..3594ea6399 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/StatisticUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/StatisticUtils.java @@ -7,12 +7,13 @@ import cz.incad.kramerius.statistics.StatisticReport; import cz.incad.kramerius.statistics.filters.DateFilter; +import cz.incad.kramerius.statistics.filters.LicenseFilter; import cz.incad.kramerius.utils.database.Offset; public class StatisticUtils { @SuppressWarnings({ "rawtypes", "unchecked" }) - public static List jdbcParams(DateFilter filter, Offset rOffset) throws ParseException { + public static List jdbcParams(DateFilter filter, LicenseFilter licenseFilter, Offset rOffset) throws ParseException { List params = new ArrayList(); if (filter.getFromDate() != null && (!filter.getFromDate().trim().equals(""))) { try { @@ -29,8 +30,15 @@ public static List jdbcParams(DateFilter filter, Offset rOffset) throws ParseExc params.add(new Timestamp(StatisticReport.DATE_FORMAT.parse(filter.getToDate()).getTime())); } } - params.add(Integer.parseInt(rOffset.getOffset())); - params.add(Integer.parseInt(rOffset.getSize())); + if (licenseFilter != null && licenseFilter.getLicence() != null) { + params.add(licenseFilter.getLicence()); + } + + if (rOffset != null) { + params.add(Integer.parseInt(rOffset.getOffset())); + params.add(Integer.parseInt(rOffset.getSize())); + } + return params; } diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/VerificationUtils.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/VerificationUtils.java new file mode 100644 index 0000000000..9ddcc29ca8 --- /dev/null +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/impl/VerificationUtils.java @@ -0,0 +1,23 @@ +package cz.incad.kramerius.statistics.impl; + +import java.text.ParseException; +import java.util.List; + +import cz.incad.kramerius.statistics.StatisticReport; + +public class VerificationUtils { + + private VerificationUtils() { + } + + public static void dateVerification(List list, String date) { + if (date != null && !"".equals(date.trim())) { + try { + StatisticReport.DATE_FORMAT.parse(date); + } catch (ParseException e) { + String dateFrom = e.getMessage(); + list.add(dateFrom); + } + } + } +} diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/utils/ReportUtils.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/utils/ReportUtils.java new file mode 100644 index 0000000000..cc0deabd30 --- /dev/null +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/utils/ReportUtils.java @@ -0,0 +1,85 @@ +package cz.incad.kramerius.statistics.utils; + +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.util.function.Consumer; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.lang3.tuple.Pair; +import org.json.JSONArray; +import org.json.JSONObject; + +import cz.incad.kramerius.statistics.filters.DateFilter; +import cz.incad.kramerius.statistics.filters.IdentifiersFilter; +import cz.incad.kramerius.statistics.filters.LicenseFilter; +import cz.incad.kramerius.statistics.filters.ModelFilter; + +public class ReportUtils { + + public static Logger LOGGER = Logger.getLogger(ReportUtils.class.getName()); + + private ReportUtils() {} + + // StringBuilder builder = new StringBuilder("q=*"); + + public static void enhanceLicense(StringBuilder builder, LicenseFilter licFilter) { + if (licFilter != null && licFilter.getLicence() != null) { + builder.append("&fq=provided_by_license:"+licFilter.getLicence()); + } + } + + public static void enhanceIdentifiers(StringBuilder builder, IdentifiersFilter iFilter) { + if (iFilter != null && iFilter.getIdentifier() != null) { + try { + String encoded = URLEncoder.encode("(all_identifiers:\""+iFilter.getIdentifier() +"\" OR all_pids:\""+iFilter.getIdentifier()+"\")", "UTF-8"); + builder.append("&fq="+encoded); + } catch (UnsupportedEncodingException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } + + } + } + + public static void enhanceDateFilter(StringBuilder builder, DateFilter dateFilter) { + if (dateFilter != null && dateFilter.getFromDate() != null && dateFilter.getToDate() != null) { + try { + String encoded = URLEncoder.encode("date:["+dateFilter.getFromDate()+" TO "+dateFilter.getToDate()+"]", "UTF-8"); + builder.append("&fq="+encoded); + } catch (UnsupportedEncodingException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } + } + } + + public static void enhanceModelFilter(StringBuilder builder, ModelFilter modelFilter) { + if (modelFilter.getModel() != null ) { + builder.append("&fq=all_models:"+modelFilter.getModel()); + } + } + + public static void facetIterate(String facetField, String resultStr, Consumer> consumer) { + JSONObject result = new JSONObject(resultStr); + if (result.has("facet_counts")) { + if (result.getJSONObject("facet_counts").has("facet_fields")) { + JSONObject facetFields = result.getJSONObject("facet_counts").getJSONObject("facet_fields"); + if (facetFields.has(facetField)) { + JSONArray jsonArray = facetFields.getJSONArray(facetField); + int i = 0; + while(i < jsonArray.length()) { + Object key = jsonArray.get(i++); + Object value = jsonArray.get(i++); + + consumer.accept(Pair.of(key, value)); + // + // + // Map map = new HashMap(); + // map.put(key.toString(), value); + // + // langs.add(map); + } + } + } + } + } +} diff --git a/shared/common/src/main/java/cz/incad/kramerius/utils/solr/SolrUtils.java b/shared/common/src/main/java/cz/incad/kramerius/utils/solr/SolrUtils.java index 1996dffa13..dace22493b 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/utils/solr/SolrUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/utils/solr/SolrUtils.java @@ -16,8 +16,12 @@ */ package cz.incad.kramerius.utils.solr; +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.InputStreamReader; import java.util.ArrayList; import java.util.List; import java.util.logging.Logger; @@ -30,11 +34,15 @@ import cz.incad.kramerius.impl.SolrAccessImpl; import cz.incad.kramerius.solr.SolrFieldsMapping; + +import org.apache.commons.io.IOUtils; +import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.auth.AuthScope; import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.client.CredentialsProvider; import org.apache.http.client.HttpResponseException; +import org.apache.http.client.methods.CloseableHttpResponse; import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.CloseableHttpClient; @@ -58,7 +66,6 @@ * @author pavels */ public class SolrUtils { - public static final Logger LOGGER = Logger.getLogger(SolrUtils.class.getName()); //public static final String DNNT_FLAG = "dnnt"; // /** Handle query */ @@ -84,6 +91,24 @@ public static XPathExpression pidPathExpr() throws XPathExpressionException { return pidPathExpr; } + public static XPathExpression ownpidPathExpr() throws XPathExpressionException { + XPathExpression pidPathExpr = fact.newXPath().compile("//str[@name='own_pid_path']"); + return pidPathExpr; + } + + + + +// /** +// * Conscturcts XPath for disecting pid path +// * @return Compiled XPath expression +// * @throws XPathExpressionException Cannot compile xpath +// */ +// public static XPathExpression pidPathExpr() throws XPathExpressionException { +// XPathExpression pidPathExpr = fact.newXPath().compile("//arr[@name='pid_paths']/str"); +// return pidPathExpr; +// } + /** * Constructs XPath for disecting PID * @return Compiled XPath expression @@ -105,7 +130,7 @@ public static XPathExpression docPidExpr() throws XPathExpressionException { * @throws XPathExpressionException Cannot compile xpath */ public static XPathExpression modelPathExpr() throws XPathExpressionException { - XPathExpression pathExpr = fact.newXPath().compile("//arr[@name='own_model_path']/str"); + XPathExpression pathExpr = fact.newXPath().compile("//str[@name='own_model_path']"); return pathExpr; } @@ -128,6 +153,28 @@ public static XPathExpression parentPidExpr() throws XPathExpressionException { XPathExpression pidExpr = fact.newXPath().compile("//arr[@name='own_parent.pid']/str"); return pidExpr; } + + /** + * Constructs XPath for disecting root PID + * @return Compiled XPath expression + * @throws XPathExpressionException Cannot compile xpath + */ + public static XPathExpression rootPidExpr() throws XPathExpressionException { + XPathExpression rootExpr = fact.newXPath().compile("//str[@name='root.pid']"); + return rootExpr; + } + + public static XPathExpression rootTitleExpr() throws XPathExpressionException { + XPathExpression rootExpr = fact.newXPath().compile("//str[@name='root.title']"); + return rootExpr; + } + + public static XPathExpression rootModelExpr() throws XPathExpressionException { + XPathExpression rootExpr = fact.newXPath().compile("//str[@name='root.model']"); + return rootExpr; + } + + /** * Constructs XPath for disecting date @@ -156,6 +203,20 @@ public static List disectPidPaths( Element element) throws XPathExpressi } } + + public static List disectOwnPidPaths( Document parseDocument) throws XPathExpressionException { + synchronized(parseDocument) { + return ownPidpath(parseDocument); + } + } + + public static List disectOwnPidPaths( Element element) throws XPathExpressionException { + synchronized(element) { + return ownPidpath(element); + } + } + + private static List paths(Node domn) throws XPathExpressionException { List list = new ArrayList<>(); NodeList paths = (NodeList) pidPathExpr().evaluate(domn, XPathConstants.NODESET); @@ -170,6 +231,20 @@ private static List paths(Node domn) throws XPathExpressionException { return new ArrayList<>(); } + private static List ownPidpath(Node domn) throws XPathExpressionException { + List list = new ArrayList<>(); + NodeList paths = (NodeList) ownpidPathExpr().evaluate(domn, XPathConstants.NODESET); + if (paths != null) { + for (int i = 0,ll=paths.getLength(); i < ll; i++) { + Node n = paths.item(i); + String text = n.getTextContent(); + list.add(text.trim()); + } + return list; + } + return new ArrayList<>(); + } + /** * Disect pid from given solr document * @param parseDocument Parsed solr document @@ -186,7 +261,40 @@ public static String disectPid(Document parseDocument) throws XPathExpressionExc return null; } } + + public static String rootPid(Document parseDocument) throws XPathExpressionException { + synchronized(parseDocument) { + Node pidNode = (Node) rootPidExpr().evaluate(parseDocument, XPathConstants.NODE); + if (pidNode != null) { + Element pidElm = (Element) pidNode; + return pidElm.getTextContent().trim(); + } + return null; + } + } + + public static String rootTitle(Document parseDocument) throws XPathExpressionException { + synchronized(parseDocument) { + Node pidNode = (Node) rootTitleExpr().evaluate(parseDocument, XPathConstants.NODE); + if (pidNode != null) { + Element pidElm = (Element) pidNode; + return pidElm.getTextContent().trim(); + } + return null; + } + } + public static String rootModel(Document parseDocument) throws XPathExpressionException { + synchronized(parseDocument) { + Node pidNode = (Node) rootModelExpr().evaluate(parseDocument, XPathConstants.NODE); + if (pidNode != null) { + Element pidElm = (Element) pidNode; + return pidElm.getTextContent().trim(); + } + return null; + } + } + public static List disectLicenses(Element topElem) { synchronized(topElem.getOwnerDocument()) { @@ -334,6 +442,23 @@ public static String disectParentPid(Document parseDocument) throws XPathExpress } } + /** + * Disect root PID from given solr document + * @param parseDocument Parsed solr document + * @return root PID + * @throws XPathExpressionException cannot disect root PID + */ + public static String disectRootPid(Document parseDocument) throws XPathExpressionException { + synchronized(parseDocument) { + Node rootPidNode = (Node) rootPidExpr().evaluate(parseDocument, XPathConstants.NODE); + if (rootPidNode != null) { + Element rootPidElm = (Element) rootPidNode; + return rootPidElm.getTextContent().trim(); + } + return null; + } + } + /** * Disect date from given solr document * @param parseDocument Parsed solr document @@ -353,8 +478,6 @@ public static String disectDate(Document parseDocument) throws XPathExpressionEx public static Document getSolrDataInternalOffset(String query, String offset) throws IOException, ParserConfigurationException, SAXException { String solrHost = KConfiguration.getInstance().getSolrHost(); String uri = solrHost +"/select?" +query+"&start="+offset+"&wt=xml"; - - InputStream inputStream = RESTHelper.inputStream(uri, "", ""); Document parseDocument = XMLUtils.parseDocument(inputStream); return parseDocument; @@ -459,8 +582,60 @@ public static String escapeQuery(String sourceQuery) { return builder.toString(); } - - + + //reads and closes entity's content stream + public static InputStream readContentAndProvideThroughBufferedStream(HttpEntity entity) throws IOException { + try (InputStream src = entity.getContent()) { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + IOUtils.copy(src, bos); + return new ByteArrayInputStream(bos.toByteArray()); + } + } + + /** + * @param query for example: q=model%3Amonograph&fl=pid%2Ctitle.search&start=0&sort=created+desc&fq=model%3Aperiodical+OR+model%3Amonograph&rows=24&hl.fragsize=20 + * i.e. url encoded and without query param wt + */ + public static InputStream requestWithTermsReturningStream(String solrHost, String query, String type) throws IOException { + String url = String.format("%s/terms?%s&wt=%s", solrHost, query, type); + HttpGet httpGet = new HttpGet(url); + CloseableHttpClient client = HttpClients.createDefault(); + try (CloseableHttpResponse response = client.execute(httpGet)) { + if (response.getStatusLine().getStatusCode() == SC_OK) { + return readContentAndProvideThroughBufferedStream(response.getEntity()); + } else { + throw new HttpResponseException(response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); + } + } + } + + /** + * @param query for example: q=model%3Amonograph&fl=pid%2Ctitle.search&start=0&sort=created+desc&fq=model%3Aperiodical+OR+model%3Amonograph&rows=24&hl.fragsize=20 + * i.e. url encoded and without query param wt + */ + public static InputStream requestWithSelectReturningStream(String solrHost, String query, String type) throws IOException { + String url = String.format("%s/select?%s&wt=%s", solrHost, query, type); + HttpGet httpGet = new HttpGet(url); + CloseableHttpClient client = HttpClients.createDefault(); + try (CloseableHttpResponse response = client.execute(httpGet)) { + if (response.getStatusLine().getStatusCode() == SC_OK) { + return readContentAndProvideThroughBufferedStream(response.getEntity()); + } else { + throw new HttpResponseException(response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); + } + } + } + + public static String requestWithSelectReturningString(String solrHost, String query, String type) throws IOException { + InputStream in = requestWithSelectReturningStream(solrHost, query, type); + BufferedReader streamReader = new BufferedReader(new InputStreamReader(in, "UTF-8")); + StringBuilder responseStrBuilder = new StringBuilder(); + String inputStr; + while ((inputStr = streamReader.readLine()) != null) { + responseStrBuilder.append(inputStr); + } + return responseStrBuilder.toString(); + } } diff --git a/shared/common/src/main/java/res/configuration.properties b/shared/common/src/main/java/res/configuration.properties index cd78dcd02a..d226ea32cd 100644 --- a/shared/common/src/main/java/res/configuration.properties +++ b/shared/common/src/main/java/res/configuration.properties @@ -505,26 +505,7 @@ legacyfs.maxActive=100 ## CDK resources -cdk.collections.sources=vc:44679769-b5bb-4ac7-ad27-a0c44698c2ea,vc:f750b424-bda4-4113-849a-5e9dbbfb5846,vc:700a6cc7-1e5c-4487-b111-c425cbc51091,vc:c4bb27af-3a51-4ac2-95c7-fd393b489e26,vc:9d2195b1-5403-4b3d-ab7c-680ac442bb73,vc:d4b466de-5435-4b76-bff7-2838bbae747b,vc:d34ba74b-026a-4c60-aee7-9250a307952c,vc:cd324f70-c034-46f1-9674-e0df4f93de86,vc:3c06120c-ffc0-4b96-b8df-80bc12e030d9,vc:b7b1b67a-25d1-4055-905d-45fedfc6a2b5,vc:4e6b7ee5-3374-4cde-9289-e1f6a2a335b2 -cdk.collections.hidden=vc:9d2195b1-5403-4b3d-ab7c-680ac442bb73 - - -cdk.collections.sources.44679769-b5bb-4ac7-ad27-a0c44698c2ea.username=jmeno -cdk.collections.sources.44679769-b5bb-4ac7-ad27-a0c44698c2ea.pswd=pswd -cdk.collections.sources.44679769-b5bb-4ac7-ad27-a0c44698c2ea.baseurl=https://abc/def - - -# Stored username and pass for library vc:44679769-b5bb-4ac7-ad27-a0c44698c2ea -#cdk.collections.sources.44679769-b5bb-4ac7-ad27-a0c44698c2ea.username=xxxx -#cdk.collections.sources.44679769-b5bb-4ac7-ad27-a0c44698c2ea.pswd=yyyy akubra.migration.logfrequency=10000 #Keycloak -keycloak.tokenurl=http://localhost:8083/auth/realms/kramerius/protocol/openid-connect/token -keycloak.clientId=krameriusClient -keycloak.secret=kyPtgyMN7rFfPiJzgaaE90cpBryAQ4nG - -#Keycloak for generating json file -keycloak.auth-server-url=http://localhost:8083/auth/ -keycloak.realm=kramerius diff --git a/shared/common/src/test/java/cz/incad/kramerius/security/SecurityExceptionTest.java b/shared/common/src/test/java/cz/incad/kramerius/security/SecurityExceptionTest.java deleted file mode 100644 index 808bde232c..0000000000 --- a/shared/common/src/test/java/cz/incad/kramerius/security/SecurityExceptionTest.java +++ /dev/null @@ -1,18 +0,0 @@ -package cz.incad.kramerius.security; - -import junit.framework.Assert; -import junit.framework.TestCase; - -public class SecurityExceptionTest extends TestCase { - - public void testSecurityMessage() { - SecurityException excp = new SecurityException( new SecurityException.SecurityExceptionInfo(SecuredActions.ADMINISTRATE)); - Assert.assertEquals("action ADMINISTRATE is not allowed", excp.getMessage()); - - excp = new SecurityException( new SecurityException.SecurityExceptionInfo(SecuredActions.READ,"uuid:xxxx")); - Assert.assertEquals("action READ is not allowed, object is uuid:xxxx", excp.getMessage()); - - excp = new SecurityException( new SecurityException.SecurityExceptionInfo(SecuredActions.READ,"uuid:xxxx", "IMG_FULL")); - Assert.assertEquals("action READ is not allowed, object is uuid:xxxx, datastream is IMG_FULL", excp.getMessage()); - } -} diff --git a/shared/common/src/test/java/cz/incad/kramerius/security/database/impl/RightTemplatesTests.java b/shared/common/src/test/java/cz/incad/kramerius/security/database/impl/RightTemplatesTests.java deleted file mode 100644 index 4a3dcad91e..0000000000 --- a/shared/common/src/test/java/cz/incad/kramerius/security/database/impl/RightTemplatesTests.java +++ /dev/null @@ -1,292 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.kramerius.security.database.impl; - -import java.io.IOException; -import java.io.StringReader; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; - -import cz.incad.kramerius.security.impl.criteria.ReadDNNTLabelsIPFiltered; -import cz.incad.kramerius.security.licenses.impl.LicenseImpl; -import org.antlr.stringtemplate.StringTemplate; -import org.apache.commons.io.IOUtils; -import org.easymock.EasyMock; -import org.junit.Ignore; -import org.junit.Test; - -import com.google.inject.Guice; -import com.google.inject.Injector; - -import cz.incad.kramerius.security.CriteriumType; -import cz.incad.kramerius.security.RightCriteriumWrapper; -import cz.incad.kramerius.security.RightCriteriumWrapperFactory; -import cz.incad.kramerius.security.Role; -import cz.incad.kramerius.security.SecuredActions; -import cz.incad.kramerius.security.User; -import cz.incad.kramerius.security.database.SecurityDatabaseUtils; -import cz.incad.kramerius.security.guice.MockGuiceSecurityModule; -import cz.incad.kramerius.security.guice.MockRightCriteriumContextGuiceMudule; -import cz.incad.kramerius.security.impl.RightCriteriumParamsImpl; -import cz.incad.kramerius.security.impl.RightImpl; -import cz.incad.kramerius.security.impl.criteria.MovingWall; -import cz.incad.kramerius.security.impl.http.MockGuiceSecurityHTTPModule; -import junit.framework.Assert; - -@Ignore -public class RightTemplatesTests { - - - - - @Test - public void testFindAllRightsWithGroupsAndCriteriums() throws IOException { - StringTemplate tmpl = SecurityDatabaseUtils.stGroup().getInstanceOf("findAllRightsWithGroupsAndCriteriums"); - - tmpl.setAttribute("userid", "1"); - tmpl.setAttribute("groupids", Arrays.asList("1", "2")); - tmpl.setAttribute("action", Arrays.asList("read")); - - tmpl.setAttribute("criteriums", Arrays.asList("cz.incad.kramerius.security.impl.criteria.ReadDNNTLabelsIPFiltered","cz.incad.kramerius.security.impl.criteria.ReadDNNTLabels")); - - String collect = IOUtils.readLines(new StringReader(tmpl.toString())).stream().map(String::trim).collect(Collectors.joining(" ")); - Assert.assertTrue(collect.equals("select * from right_entity ent left join rights_criterium_entity crit on (ent.rights_crit=crit.crit_id) left join criterium_param_entity param on (crit.citeriumparam=param.crit_param_id) left join labels_entity lbl on (crit.label_id=lbl.label_id) left join user_entity users on (ent.user_id = users.user_id) left join group_entity groups on (ent.group_id = groups.group_id) where (ent.\"user_id\"=1 or ent.\"group_id\" in (1 ,2 ) ) and \"qname\" in ('cz.incad.kramerius.security.impl.criteria.ReadDNNTLabelsIPFiltered','cz.incad.kramerius.security.impl.criteria.ReadDNNTLabels') and \"action\"='read'")); - - - tmpl = SecurityDatabaseUtils.stGroup().getInstanceOf("findAllRightsWithGroupsAndCriteriums"); - - tmpl.setAttribute("userid", "1"); - tmpl.setAttribute("groupids", Arrays.asList("1", "2")); - tmpl.setAttribute("action", Arrays.asList("read")); - - collect = IOUtils.readLines(new StringReader(tmpl.toString())).stream().map(String::trim).collect(Collectors.joining(" ")); - Assert.assertTrue(collect.equals("select * from right_entity ent left join rights_criterium_entity crit on (ent.rights_crit=crit.crit_id) left join criterium_param_entity param on (crit.citeriumparam=param.crit_param_id) left join labels_entity lbl on (crit.label_id=lbl.label_id) left join user_entity users on (ent.user_id = users.user_id) left join group_entity groups on (ent.group_id = groups.group_id) where (ent.\"user_id\"=1 or ent.\"group_id\" in (1 ,2 ) ) and \"action\"='read'")); - } - - - @Test - public void testFindRights() { - StringTemplate tmpl = SecurityDatabaseUtils.stGroup().getInstanceOf("findAllRights"); - - Map> m = new HashMap>(); - - - m.put("gname", Arrays.asList("k4_admins", "common_users")); - m.put("action", Arrays.asList("read", "store")); - m.put("uuid", Arrays.asList("uuid:112233", "uuid:223344")); - - tmpl.setAttribute("params", m); - - String expectedSQL = "select * from right_entity ent\n" - + "left join rights_criterium_entity crit on (ent.rights_crit=crit.crit_id) left join criterium_param_entity param on (crit.citeriumparam=param.crit_param_id) left join labels_entity lbl on (crit.label_id=lbl.label_id) left join user_entity users on (ent.user_id = users.user_id) left join group_entity groups on (ent.group_id = groups.group_id)\n" - + " where (action in ('read','store')) and (gname in ('k4_admins','common_users')) and (uuid in ('uuid:112233','uuid:223344')) "; - - String templateString = tmpl.toString(); - - String expectedSQLReplaced = expectedSQL.replaceAll("\\s+", " "); - String templateStringReplaced = templateString.replaceAll("\\s+", " "); - Assert.assertEquals(expectedSQLReplaced, templateStringReplaced); - } - - - @Test - public void testFindRights_SomeParams() { - StringTemplate tmpl = SecurityDatabaseUtils.stGroup().getInstanceOf("findAllRights"); - - Map> m = new HashMap>(); - - m.put("gname", Arrays.asList("k4_admins", "common_users")); - tmpl.setAttribute("params", m); - String expectedSQL = "select * from right_entity ent\n" - + "left join rights_criterium_entity crit on (ent.rights_crit=crit.crit_id) left join criterium_param_entity param on (crit.citeriumparam=param.crit_param_id) left join labels_entity lbl on (crit.label_id=lbl.label_id) left join user_entity users on (ent.user_id = users.user_id) left join group_entity groups on (ent.group_id = groups.group_id)\n" - + " where (gname in ('k4_admins','common_users')) "; - - String templateString = tmpl.toString(); - Assert.assertEquals(expectedSQL.replaceAll("\\s+", " "), templateString.replaceAll("\\s+", " ")); - } - - @Test - public void testFindRights_NoParams() { - StringTemplate tmpl = SecurityDatabaseUtils.stGroup().getInstanceOf("findAllRights"); - tmpl.setAttribute("params", null); - String expectedSQL = "select * from right_entity ent\n" - + "left join rights_criterium_entity crit on (ent.rights_crit=crit.crit_id) left join criterium_param_entity param on (crit.citeriumparam=param.crit_param_id) left join labels_entity lbl on (crit.label_id=lbl.label_id) left join user_entity users on (ent.user_id = users.user_id) left join group_entity groups on (ent.group_id = groups.group_id)\n" - + " "; - String templateString = tmpl.toString(); - Assert.assertEquals(expectedSQL.replaceAll("\\s+", " "), templateString.replaceAll("\\s+" , " ")); - - } - - @Test - public void testInsertCriterium() { - Injector injector = injector(); - RightCriteriumWrapperFactory wrapperFactory = injector.getInstance(RightCriteriumWrapperFactory.class); - RightCriteriumWrapper mw = wrapperFactory.createCriteriumWrapper(MovingWall.class.getName()); - - StringTemplate template1 = SecurityDatabaseUtils.stGroup().getInstanceOf("insertRightCriterium"); - template1.setAttribute("criteriumWrapper", mw); - template1.setAttribute("type", mw.getCriteriumType().getVal()); - - // no params - String sql1 = template1.toString(); - String expectedSql = " insert into rights_criterium_entity(crit_id,qname,\"type\", label_id)\n" - + " values(nextval('crit_id_sequence'),\n" - + " 'cz.incad.kramerius.security.impl.criteria.MovingWall',\n" + " 1, NULL ) "; - Assert.assertEquals(expectedSql.replaceAll("\\s+", "").trim(), sql1.replaceAll("\\s+","").trim()); - - - - RightCriteriumParamsImpl paramsImpl = new RightCriteriumParamsImpl(2); - paramsImpl.setObjects(new String[] { "1", "2", "3" }); - mw.setCriteriumParams(paramsImpl); - StringTemplate template2 = SecurityDatabaseUtils.stGroup().getInstanceOf("insertRightCriterium"); - template2.setAttribute("criteriumWrapper", mw); - template2.setAttribute("type", mw.getCriteriumType().getVal()); - String sql2 = template2.toString(); - String expectedSql2 = " \n" - + " insert into rights_criterium_entity(crit_id,qname, \"type\",citeriumparam, label_id)\n" - + " values(nextval('crit_id_sequence'),\n" - + " 'cz.incad.kramerius.security.impl.criteria.MovingWall',\n" + " 1,\n" - + " 2, NULL ) "; - - Assert.assertEquals(expectedSql2.replaceAll("\\s+",""), sql2.replaceAll("\\s+", "")); - - - } - - - @Test - public void testInsertCriteriumLabel2() { - Injector injector = injector(); - RightCriteriumWrapperFactory wrapperFactory = injector.getInstance(RightCriteriumWrapperFactory.class); - RightCriteriumWrapper lb = wrapperFactory.createCriteriumWrapper(ReadDNNTLabelsIPFiltered.class.getName()); - - RightCriteriumParamsImpl paramsImpl = new RightCriteriumParamsImpl(2); - paramsImpl.setObjects(new String[] { "1", "2", "3" }); - lb.setCriteriumParams(paramsImpl); - - StringTemplate template1 = SecurityDatabaseUtils.stGroup().getInstanceOf("insertRightCriterium"); - template1.setAttribute("criteriumWrapper", lb); - template1.setAttribute("type", lb.getCriteriumType().getVal()); - - String sql1 = template1.toString(); - String expected = " insert into rights_criterium_entity(crit_id,qname, \"type\",citeriumparam, label_id)\n" + - " values(nextval('crit_id_sequence'),\n" + - " 'cz.incad.kramerius.security.impl.criteria.ReadDNNTLabelsIPFiltered',\n" + - " 1,\n" + - " 2\n" + - " , NULL )\n"; - - Assert.assertEquals(expected.replaceAll("\\s+", "").trim(), sql1.replaceAll("\\s+", "").trim()); - } - - @Test - public void testInsertCriteriumLabel() { - Injector injector = injector(); - RightCriteriumWrapperFactory wrapperFactory = injector.getInstance(RightCriteriumWrapperFactory.class); - RightCriteriumWrapper lb = wrapperFactory.createCriteriumWrapper(ReadDNNTLabelsIPFiltered.class.getName()); - - lb.setLabel(new LicenseImpl(4, "name","desc","group")); - - StringTemplate template1 = SecurityDatabaseUtils.stGroup().getInstanceOf("insertRightCriterium"); - template1.setAttribute("criteriumWrapper", lb); - template1.setAttribute("type", lb.getCriteriumType().getVal()); - - String sql1 = template1.toString(); - Assert.assertEquals("insert into rights_criterium_entity(crit_id,qname,\"type\", label_id) values(nextval('crit_id_sequence'), 'cz.incad.kramerius.security.impl.criteria.ReadDNNTLabelsIPFiltered', 1, 4 )".replaceAll("\\s+","").trim(), sql1.replaceAll("\\s+","").trim()); - - - RightCriteriumParamsImpl paramsImpl = new RightCriteriumParamsImpl(2); - paramsImpl.setObjects(new String[] { "1", "2", "3" }); - lb.setCriteriumParams(paramsImpl); - String sql2 = template1.toString(); - - String expected = "insert into rights_criterium_entity(crit_id,qname, \"type\",citeriumparam, label_id)\n" + - " values(nextval('crit_id_sequence'),\n" + - " 'cz.incad.kramerius.security.impl.criteria.ReadDNNTLabelsIPFiltered',\n" + - " 1,\n" + - " 2\n" + - " , 4 )"; - - - Assert.assertEquals(sql2.replaceAll("\\s+", "").trim(),expected.replaceAll("\\s+", "").trim()); - - } - - @Test - public void testInsertRightCriteriumParamsTemplate() { - RightCriteriumParamsImpl paramsImpl = new RightCriteriumParamsImpl(2); - paramsImpl.setObjects(new String[] { "1", "2", "3" }); - paramsImpl.setShortDescription("short desc"); - - StringTemplate template = SecurityDatabaseUtils.stGroup().getInstanceOf("insertRightCriteriumParams"); - template.setAttribute("params", paramsImpl); - String sql = template.toString(); - String expectedSql = " insert into criterium_param_entity(crit_param_id,short_desc,long_desc, vals) \n" - + " values(\n" + " nextval('crit_param_id_sequence'),\n" + " 'short desc',\n" - + " '',\n" + " '1;2;3'\n" + " )"; - - Assert.assertEquals(expectedSql.replaceAll("\\s+", " "), sql.replaceAll("\\s+"," ")); - } - - - - @Test - public void testInsertRightTemplate() { - Injector injector = injector(); - - RightCriteriumParamsImpl paramsImpl = new RightCriteriumParamsImpl(2); - paramsImpl.setObjects(new String[] { "1", "2", "3" }); - paramsImpl.setShortDescription("shortDesc"); - - RightCriteriumWrapperFactory wrapperFactory = injector.getInstance(RightCriteriumWrapperFactory.class); - RightCriteriumWrapper mw = wrapperFactory.loadExistingWrapper(CriteriumType.CLASS, MovingWall.class.getName(), - 5, null); - mw.setCriteriumParams(paramsImpl); - - Role mockUser = EasyMock.createMock(Role.class); - EasyMock.expect(mockUser.getId()).andReturn(111); - - RightImpl rightImpl = new RightImpl(1, mw, "0xABC", SecuredActions.READ.getFormalName(), mockUser); - rightImpl.setCriteriumWrapper(mw); - - StringTemplate template = SecurityDatabaseUtils.stGroup().getInstanceOf("insertRight"); - template.setAttribute("association", rightImpl.getRole() instanceof Role ? "group_id" : "user_id"); - template.setAttribute("right", rightImpl); - template.setAttribute("priority", - rightImpl.getFixedPriority() == 0 ? "NULL" : "" + rightImpl.getFixedPriority()); - String sql = template.toString(); - - String expectedSql = " \n" - + " insert into right_entity(right_id,uuid,action,rights_crit,\"user_id\", fixed_priority) \n" - + " values(\n" + " nextval('right_id_sequence'),\n" + " 'uuid:0xABC',\n" - + " 'read',\n" + " 5,\n" + " 0,\n" + " NULL\n" - + " ) "; - - Assert.assertEquals(expectedSql.replaceAll("\\s+"," "), sql.replaceAll("\\s+", " ")); - } - - - protected Injector injector() { - return Guice.createInjector(new MockGuiceSecurityModule(), new MockGuiceSecurityHTTPModule(), - new MockRightCriteriumContextGuiceMudule()); - } - -} diff --git a/shared/common/src/test/java/cz/incad/kramerius/security/impl/RightCriteriumLoaderImplTest.java b/shared/common/src/test/java/cz/incad/kramerius/security/impl/RightCriteriumLoaderImplTest.java deleted file mode 100644 index 2be7121509..0000000000 --- a/shared/common/src/test/java/cz/incad/kramerius/security/impl/RightCriteriumLoaderImplTest.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright (C) 2010 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.kramerius.security.impl; - -import java.util.List; - -import org.junit.Assert; -import org.junit.Before; -import org.junit.Test; - -import cz.incad.kramerius.security.RightCriterium; - -public class RightCriteriumLoaderImplTest { - - @Before - public void setup() { - - } - - @Test - public void testRightCriterium() { - ClassRightCriteriumLoaderImpl rr = new ClassRightCriteriumLoaderImpl(); - List criteriums = rr.getCriteriums(); - Assert.assertFalse(criteriums.isEmpty()); - } -} diff --git a/shared/common/src/test/java/cz/incad/kramerius/security/impl/http/MockGuiceSecurityHTTPModule.java b/shared/common/src/test/java/cz/incad/kramerius/security/impl/http/MockGuiceSecurityHTTPModule.java index 81c2a859be..9cc360bf0d 100644 --- a/shared/common/src/test/java/cz/incad/kramerius/security/impl/http/MockGuiceSecurityHTTPModule.java +++ b/shared/common/src/test/java/cz/incad/kramerius/security/impl/http/MockGuiceSecurityHTTPModule.java @@ -32,7 +32,7 @@ public class MockGuiceSecurityHTTPModule extends AbstractModule { protected void configure() { RightsReturnObject rightsReturnObject = new RightsReturnObject(null, EvaluatingResultState.FALSE); RightsResolver isAllowed = EasyMock.createMock(RightsResolver.class); - EasyMock.expect(isAllowed.isActionAllowed(SecuredActions.READ.getFormalName(), FedoraUtils.IMG_FULL_STREAM, SpecialObjects.REPOSITORY.getPid(), new ObjectPidsPath(SpecialObjects.REPOSITORY.getPid()))).andReturn(rightsReturnObject); + EasyMock.expect(isAllowed.isActionAllowed(SecuredActions.A_READ.getFormalName(), FedoraUtils.IMG_FULL_STREAM, SpecialObjects.REPOSITORY.getPid(), new ObjectPidsPath(SpecialObjects.REPOSITORY.getPid()))).andReturn(rightsReturnObject); replay(isAllowed); bind(RightsResolver.class).toInstance(isAllowed); diff --git a/shared/common/src/test/java/cz/incad/kramerius/statistics/impl/NKPLogReportTest.java b/shared/common/src/test/java/cz/incad/kramerius/statistics/impl/NKPLogReportTest.java deleted file mode 100644 index 9ff704db0a..0000000000 --- a/shared/common/src/test/java/cz/incad/kramerius/statistics/impl/NKPLogReportTest.java +++ /dev/null @@ -1,136 +0,0 @@ -package cz.incad.kramerius.statistics.impl; - -import cz.incad.kramerius.statistics.ReportedAction; -import cz.incad.kramerius.statistics.accesslogs.database.DatabaseStatisticsAccessLogImpl; -import junit.framework.Assert; -import org.antlr.stringtemplate.StringTemplate; -import org.json.JSONArray; -import org.json.JSONObject; -import org.junit.Ignore; -import org.junit.Test; - -import java.io.UnsupportedEncodingException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.time.LocalDateTime; -import java.time.ZoneId; -import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.logging.Level; - -public class NKPLogReportTest { - - @Test - public void testTemplate() { - StringTemplate statRecord = DatabaseStatisticsAccessLogImpl.stGroup.getInstanceOf("nkpLogsReport"); - statRecord.setAttribute("action", "READ"); - statRecord.setAttribute("action", ReportedAction.READ.name()); - statRecord.setAttribute("fromDefined", true); - statRecord.setAttribute("toDefined", true); - Assert.assertNotNull(statRecord.toString()); - Assert.assertTrue(statRecord.toString().contains("date")); - Assert.assertTrue(statRecord.toString().contains(">")); - Assert.assertTrue(statRecord.toString().contains("?")); - } - - @Test - @Ignore - public void testProcessDatabase() throws SQLException { - String url = "jdbc:postgresql://localhost/kramerius4-edu"; - Properties props = new Properties(); - props.setProperty("user","fedoraAdmin"); - props.setProperty("password","fedoraAdmin"); - //props.setProperty("ssl","true"); - Connection conn = DriverManager.getConnection(url, props); - - String sql = "select \n" + - "\tsl.record_id as slrecord_id, \n" + - "\tsl.pid as slpid, \n" + - "\tsl.date as sldate,\n" + - "\tsl.remote_ip_address as slremote_ip_address, \n" + - "\tsl.\"USER\" as sluser, \n" + - "\tsl.stat_action as slstat_action,\n" + - "\tsl.dnnt as sldnnt,\n" + - "\tsl.providedbydnnt as slprovidedbydnnt,\n" + - "\tsl.evaluatemap as slevaluatemap,\n" + - "\tsl.usersessionattributes as slusersessionattributes,\n" + - "\t\n" + - "\tsd.detail_id as sddetail_id,\n" + - "\tsd.pid as sdpid,\n" + - "\tsd.model as sdmodel,\n" + - "\tsd.issued_date as sdissued_date,\n" + - "\tsd.solr_date as sdsolr_date,\n" + - "\tsd.rights as sdrights,\n" + - "\tsd.lang as sdlang,\n" + - "\tsd.title as sdtitle,\n" + - "\tsd.branch_id as sdbranch_id,\n" + - "\t\n" + - "\tsa.author_id as saauthor_id,\n" + - "\tsa.author_name as saauthor_name,\n" + - "\n" + - "\tsp.publisher_id as sppublisher_id,\n" + - "\tsp.publisher_name as sppublisher_name\n" + - "\t\n" + - "\tfrom statistics_access_log sl\n" + - "join statistic_access_log_detail sd using(record_id)\n" + - "left join statistic_access_log_detail_authors sa using(detail_id)\n" + - "left join statistic_access_log_detail_publishers sp using(detail_id)\n" + - "where date>'2021-01-02' AND date <'2021-01-03'\n" + - "order by sl.record_id, sd.detail_id, sd.branch_id, sa.author_id\n"; - - long start = System.currentTimeMillis(); - final AtomicInteger counter = new AtomicInteger(0); -// -// System.out.println(sql); -// NKPLogReport.statisticsIterate(new ArrayList(),sql, conn, (record)->{ -// if (record.get("branches").size() > 1) { -// System.out.println(record); -// } -// counter.incrementAndGet(); -// }); - - new NKPLogReport.StastisticsIteration(sql, new ArrayList(),conn, record-> { - //logReport(collectedRecord, sup); - System.out.println(record); - }).iterate(); - - long stop = System.currentTimeMillis(); - System.out.println("It took "+(stop - start)+" ms "); - } - - @Test - public void testNull() { - - - - long start = System.currentTimeMillis(); - for (int i = 0; i < 200000; i++) { - - LocalDateTime localDateTime = new Date().toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime(); - localDateTime = localDateTime.plusDays(1); - - hashVal("xxxsadlfjsdlakjflkxcxlkjlkjkljdfdf"); - if (i % 10000 == 0) { - System.out.println("Test "+i); - } - } - long stop = System.currentTimeMillis(); - System.out.println((stop - start)+" ms "); - } - - public static String hashVal(String value) { - try { - MessageDigest md5 = MessageDigest.getInstance("MD5"); - Base64.Encoder base64Encoder = Base64.getEncoder(); - md5.update(value.getBytes("UTF-8")); - return base64Encoder.encodeToString(md5.digest()); - } catch (NoSuchAlgorithmException | UnsupportedEncodingException e) { - //LOGGER.log(Level.SEVERE, e.getMessage(),e); - return value; - } - } - -} From 6e85741bbe7fdf1ade39d95c039d15b0a3b9b8eb Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 29 Dec 2022 12:54:20 +0100 Subject: [PATCH 046/235] CDK update/timestamps/refresh from registr --- .../rest/api/guice/ApiServletModule.java | 6 +- .../virtualcollection/ClientResources.java | 122 -------- .../ClientVirtualCollections.java | 215 -------------- .../v10/proxy/ConnectedInfoResource.java | 159 +++++----- .../apiNew/client/v60/libs/Instances.java | 27 +- .../DefaultPropertiesInstances.java | 201 +++++++------ .../Kramerius/backend/guice/BaseModule.java | 4 - .../CDKVirtualCollectionsGet.java | 14 - .../impl/CDKResourcesFilter.java | 47 --- .../impl/CDKVirtualCollectionsGetImpl.java | 275 ------------------ 10 files changed, 197 insertions(+), 873 deletions(-) delete mode 100644 rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientResources.java delete mode 100644 rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientVirtualCollections.java delete mode 100644 shared/common/src/main/java/cz/incad/kramerius/virtualcollections/CDKVirtualCollectionsGet.java delete mode 100644 shared/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKResourcesFilter.java delete mode 100644 shared/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java index 624cd4e224..fdd4a3f699 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java @@ -55,8 +55,6 @@ import cz.incad.kramerius.rest.api.k5.client.pdf.PDFResource; import cz.incad.kramerius.rest.api.k5.client.rights.ClientRightsResource; import cz.incad.kramerius.rest.api.k5.client.search.SearchResource; -//import cz.incad.kramerius.rest.api.k5.client.user.ClientUserResource; -import cz.incad.kramerius.rest.api.k5.client.virtualcollection.ClientResources; import cz.incad.kramerius.rest.apiNew.client.v60.ClientProvider; import cz.incad.kramerius.rest.apiNew.client.v60.ClientUserResource; import cz.incad.kramerius.rest.apiNew.client.v60.filter.DefaultFilter; @@ -65,7 +63,6 @@ import cz.incad.kramerius.rest.apiNew.client.v60.libs.properties.DefaultPropertiesInstances; import cz.incad.kramerius.timestamps.TimestampStore; import cz.incad.kramerius.timestamps.impl.SolrTimestampStore; -import cz.incad.kramerius.rest.api.k5.client.virtualcollection.ClientVirtualCollections; import cz.incad.kramerius.rest.api.processes.LRResource; import cz.incad.kramerius.rest.api.serialization.SimpleJSONMessageBodyReader; import cz.incad.kramerius.rest.api.serialization.SimpleJSONMessageBodyWriter; @@ -91,8 +88,7 @@ protected void configureServlets() { // API Client 5.0 Resources bind(ClientUserResource.class); bind(FeederResource.class); - bind(ClientVirtualCollections.class); - bind(ClientResources.class); + bind(SearchResource.class); bind(FeedbackResource.class); bind(ClientRightsResource.class); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientResources.java b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientResources.java deleted file mode 100644 index 61251ca747..0000000000 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientResources.java +++ /dev/null @@ -1,122 +0,0 @@ -package cz.incad.kramerius.rest.api.k5.client.virtualcollection; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; - -import javax.ws.rs.Consumes; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import javax.xml.xpath.XPathExpressionException; - -import org.w3c.dom.Document; - -import org.json.JSONObject; -import org.json.JSONArray; - - -import com.google.inject.Inject; -import com.google.inject.name.Named; - -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.rest.api.exceptions.GenericApplicationException; -import cz.incad.kramerius.rest.api.replication.exceptions.ObjectNotFound; -import cz.incad.kramerius.virtualcollections.Collection; -import cz.incad.kramerius.virtualcollections.Collection.Description; -import cz.incad.kramerius.virtualcollections.CollectionsManager; -import cz.incad.kramerius.virtualcollections.impl.CDKResourcesFilter; -import cz.incad.kramerius.virtualcollections.impl.CDKVirtualCollectionsGetImpl; - -@Path("/v5.0/sources") -public class ClientResources { - - public static Logger LOGGER = Logger.getLogger(ClientResources.class.getName()); - - @Inject - @Named("fedora") - CollectionsManager colManager; - - @Inject - @Named("securedFedoraAccess") - FedoraAccess fedoraAccess; - - private CDKResourcesFilter cdkResFilter = new CDKResourcesFilter(); - - - @GET - @Path("{pid}") - @Consumes - @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") - public Response oneVirtualCollection(@PathParam("pid") String pid) { - try { - Collection vc = this.colManager.getCollection(pid); - if (vc != null && this.cdkResFilter.isResource(vc.getPid())) { - if (!this.cdkResFilter.isHidden(vc.getPid())) { - return Response - .ok() - .entity(resourceTOJSON(this.fedoraAccess, vc)).build(); - } else { - throw new ObjectNotFound("cannot find vc '" + pid + "'"); - } - } else { - throw new ObjectNotFound("cannot find vc '" + pid + "'"); - } - } catch (ObjectNotFound e) { - throw e; - } catch (Exception e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - throw new GenericApplicationException(e.getMessage()); - } - } - - @GET - @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") - public Response get() { - try { - JSONArray jsonArr = new JSONArray(); - List resources = this.cdkResFilter.getResources(); - for (String pid : resources) { - if (!this.cdkResFilter.isHidden(pid)) { - Collection col = this.colManager.getCollection(pid); - if (col != null) { - jsonArr.put(resourceTOJSON(this.fedoraAccess, col)); - } - } - } - return Response.ok().entity(jsonArr.toString()).build(); - } catch (Exception e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - throw new GenericApplicationException(e.getMessage()); - } - } - - public static JSONObject resourceTOJSON(FedoraAccess fa, Collection vc) throws XPathExpressionException, IOException { - JSONObject jsonObj = new JSONObject(); - jsonObj.put("pid", vc.getPid()); - jsonObj.put("label", vc.getLabel()); - - Document dc = fa.getDC(vc.getPid()); - String url = CDKVirtualCollectionsGetImpl.disectURL(dc); - if (url != null) { - jsonObj.put("url", url); - } - - JSONObject jsonMap = new JSONObject(); - List descriptions = vc.getDescriptions(); - for (Description description : descriptions) { - jsonMap.put(description.getLangCode(), description.getText()); - } - jsonObj.put("descs", jsonMap); - - return jsonObj; - } - -} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientVirtualCollections.java b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientVirtualCollections.java deleted file mode 100644 index 00875dfbc8..0000000000 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/virtualcollection/ClientVirtualCollections.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright (C) 2013 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -package cz.incad.kramerius.rest.api.k5.client.virtualcollection; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.Collections; -import java.util.List; -import java.util.Locale; -import java.util.logging.Level; -import java.util.logging.Logger; - -import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.Consumes; -import javax.ws.rs.GET; -import javax.ws.rs.Path; -import javax.ws.rs.PathParam; -import javax.ws.rs.Produces; -import javax.ws.rs.QueryParam; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; - -import org.json.JSONArray; - -import com.google.inject.Inject; -import com.google.inject.Provider; -import com.google.inject.name.Named; - -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.imaging.ImageStreams; -import cz.incad.kramerius.rest.api.exceptions.GenericApplicationException; -import cz.incad.kramerius.rest.api.k5.client.item.exceptions.PIDNotFound; -import cz.incad.kramerius.rest.api.replication.exceptions.ObjectNotFound; -import cz.incad.kramerius.utils.ApplicationURL; -import cz.incad.kramerius.virtualcollections.CDKVirtualCollectionsGet; -import cz.incad.kramerius.virtualcollections.Collection; -import cz.incad.kramerius.virtualcollections.CollectionUtils; -import cz.incad.kramerius.virtualcollections.CollectionsManager; -import cz.incad.kramerius.virtualcollections.CollectionsManager.SortOrder; -import cz.incad.kramerius.virtualcollections.CollectionsManager.SortType; - - -@Path("/v5.0/vc") -public class ClientVirtualCollections { - - public static final Logger LOGGER = Logger - .getLogger(ClientVirtualCollections.class.getName()); - - - @Inject - @Named("fedora") - CollectionsManager manager; - - @Inject - @Named("securedFedoraAccess") - FedoraAccess fedoraAccess; - - @Inject - CDKVirtualCollectionsGet cdkVirtGet; - - - Provider req; - - @GET - @Path("{pid}") - @Consumes - @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") - public Response oneVirtualCollection(@PathParam("pid") String pid) { - try { - String resource = this.cdkVirtGet.getResource(pid); - Collection col = this.cdkVirtGet.virtualCollectionsFromResource(pid, resource); - return Response - .ok() - .entity(CollectionUtils - .virtualCollectionTOJSON(col)).build(); - - - } catch (ObjectNotFound e) { - throw e; - } catch (Exception e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - throw new GenericApplicationException(e.getMessage()); - } - } - - private void checkPid(String pid) throws PIDNotFound { - try { - if (!this.fedoraAccess.isObjectAvailable(pid)) { - throw new PIDNotFound("pid not found"); - } - } catch (IOException e) { - throw new PIDNotFound("pid not found"); - } catch(Exception e) { - throw new PIDNotFound("error while parsing pid ("+pid+")"); - } - } - - - @GET - @Path("{pid}/thumb") - public Response thumb(@PathParam("pid") String pid) { - try { - checkPid(pid); - String suri = ApplicationURL - .applicationURL(this.req.get()) - + "/api/v5.0/item/" + pid + "/streams/"+ImageStreams.IMG_THUMB; - URI uri = new URI(suri); - return Response.temporaryRedirect(uri).build(); - } catch (URISyntaxException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - throw new PIDNotFound("pid not found '" + pid + "'"); - } - } - - - @GET - @Path("{pid}/full") - public Response full(@PathParam("pid") String pid) { - try { - checkPid(pid); - String suri = ApplicationURL - .applicationURL(this.req.get()) - + "/api/v5.0/item/" + pid + "/streams/"+ImageStreams.IMG_FULL; - URI uri = new URI(suri); - return Response.temporaryRedirect(uri).build(); - } catch (URISyntaxException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - throw new PIDNotFound("pid not found '" + pid + "'"); - } - } - - - @GET - @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") - public Response get( - @QueryParam("sort") String ordering, - @QueryParam("sortType") String sType, - @QueryParam("langCode") String langCode) { - try { - - SortOrder order = sortOrdering(ordering); - SortType sortType = sortType(sType); - List collections = null; - if (order != null) { - if (sortType == null) { - sortType = SortType.ALPHABET; - } - Locale locale = null; - if (langCode != null) { - locale = Locale.forLanguageTag(langCode); - } else { - locale = this.req.get().getLocale(); - } - collections = this.manager.getSortedCollections(locale, order, sortType); - } else { - collections = this.manager.getCollections(); - } - - JSONArray jsonArr = new JSONArray(); - for (Collection vc : collections) { - jsonArr.put(CollectionUtils - .virtualCollectionTOJSON(vc)); - } - return Response.ok().entity(jsonArr.toString()).build(); - } catch (Exception e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - throw new GenericApplicationException(e.getMessage()); - } - } - - private SortType sortType(String sType) { - if (sType != null) { - SortType selVal = null; - SortType[] values = CollectionsManager.SortType.values(); - for (SortType sortType : values) { - if (sortType.name().equals(sType)) { - selVal = sortType; - break; - } - - } - return selVal; - } - return null; - } - - private SortOrder sortOrdering(String sortOrdering) { - if (sortOrdering!=null) { - SortOrder selectedVal = null; - for (SortOrder v : CollectionsManager.SortOrder.values()) { - if (sortOrdering.equals(v.name())) { - selectedVal = v; - break; - } - } - return selectedVal; - } else return null; - } - -} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java index 73d6067410..bfd84be18b 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java @@ -40,111 +40,120 @@ @Path("/admin/v7.0/connected") public class ConnectedInfoResource { - public static final Logger LOGGER = Logger.getLogger(ConnectedInfoResource.class.getName()); - - @Inject - private Instances libraries; - - @Inject - private TimestampStore timestampStore; - + public static final Logger LOGGER = Logger.getLogger(ConnectedInfoResource.class.getName()); + + @Inject + private Instances libraries; + + @Inject + private TimestampStore timestampStore; + @GET @Produces(MediaType.APPLICATION_JSON) public Response getAllConnected() { - JSONObject retval = new JSONObject(); - this.libraries.allInstances().forEach(library-> { - JSONObject json = libraryJSON(library); - retval.put(library.getName(), json); - }); + JSONObject retval = new JSONObject(); + this.libraries.allInstances().forEach(library -> { + JSONObject json = libraryJSON(library); + retval.put(library.getName(), json); + }); return Response.ok(retval).build(); } private JSONObject libraryJSON(OneInstance found) { - JSONObject retval = new JSONObject(); - retval.put("status", found.isConnected()); - retval.put("type", found.getType().name()); - return retval; + JSONObject retval = new JSONObject(); + retval.put("status", found.isConnected()); + retval.put("type", found.getType().name()); + return retval; } - - @GET + @GET @Path("{library}/status") @Produces(MediaType.APPLICATION_JSON) public Response library(@PathParam("library") String library) { - OneInstance find = this.libraries.find(library); - if (find != null) { - return Response.ok(libraryJSON(find)).build(); - } else { - return Response.status(Response.Status.BAD_REQUEST).build(); - } + OneInstance find = this.libraries.find(library); + if (find != null) { + return Response.ok(libraryJSON(find)).build(); + } else { + return Response.status(Response.Status.BAD_REQUEST).build(); + } } - @PUT + @PUT @Path("{library}/status") @Produces(MediaType.APPLICATION_JSON) - public Response library(@PathParam("library") String library,@QueryParam("status") String status) { - OneInstance find = this.libraries.find(library); - if (find != null) { - find.setConnected(Boolean.parseBoolean(status), TypeOfChangedStatus.user); - return Response.ok(libraryJSON(find)).build(); - } else { - return Response.status(Response.Status.BAD_REQUEST).build(); - } + public Response library(@PathParam("library") String library, @QueryParam("status") String status) { + OneInstance find = this.libraries.find(library); + if (find != null) { + find.setConnected(Boolean.parseBoolean(status), TypeOfChangedStatus.user); + return Response.ok(libraryJSON(find)).build(); + } else { + return Response.status(Response.Status.BAD_REQUEST).build(); + } } - - - @GET + @GET @Path("{library}/timestamp") @Produces(MediaType.APPLICATION_JSON) public Response timestamp(@PathParam("library") String library) { - try { - Timestamp latest = this.timestampStore.findLatest(library); - if (latest != null) { - return Response.ok(latest.toJSONObject().toString()).build(); - } else return Response.status(Response.Status.NOT_FOUND).build(); - } catch (SolrServerException | IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); - } - + try { + Timestamp latest = this.timestampStore.findLatest(library); + if (latest != null) { + return Response.ok(latest.toJSONObject().toString()).build(); + } else + return Response.status(Response.Status.NOT_FOUND).build(); + } catch (SolrServerException | IOException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); + } + } - - @GET + @GET + @Path("refresh") + @Produces(MediaType.APPLICATION_JSON) + public Response refresh() { + try { + this.libraries.cronRefresh(); + return Response.ok(new JSONObject()).build(); + } catch (Exception e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); + } + } + + @GET @Path("{library}/timestamps") @Produces(MediaType.APPLICATION_JSON) public Response timestamps(@PathParam("library") String library) { - try { - JSONArray array = new JSONArray(); - List retrieveTimestamp = this.timestampStore.retrieveTimestamps(library); - retrieveTimestamp.forEach(t-> { - array.put(t.toJSONObject()); - }); - return Response.ok(array.toString()).build(); - } catch (SolrServerException | IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); - } - + try { + JSONArray array = new JSONArray(); + List retrieveTimestamp = this.timestampStore.retrieveTimestamps(library); + retrieveTimestamp.forEach(t -> { + array.put(t.toJSONObject()); + }); + return Response.ok(array.toString()).build(); + } catch (SolrServerException | IOException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); + } } - @PUT + @PUT @Path("{library}/timestamp") @Produces(MediaType.APPLICATION_JSON) - @Consumes({MediaType.APPLICATION_JSON,MediaType.APPLICATION_JSON+";charset=utf-8"}) + @Consumes({ MediaType.APPLICATION_JSON, MediaType.APPLICATION_JSON + ";charset=utf-8" }) public Response timestamp(@PathParam("library") String library, JSONObject jsonObject) { - try { - Timestamp timestamp = SolrTimestamp.fromJSONDoc(library, jsonObject); - if (timestamp.getDate() == null) { - timestamp.updateDate(new Date()); - } - timestamp.updateName(library); - this.timestampStore.storeTimestamp(timestamp); - return Response.ok(timestamp.toJSONObject().toString()).build(); - } catch (SolrServerException | IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); - } + try { + Timestamp timestamp = SolrTimestamp.fromJSONDoc(library, jsonObject); + if (timestamp.getDate() == null) { + timestamp.updateDate(new Date()); + } + timestamp.updateName(library); + this.timestampStore.storeTimestamp(timestamp); + return Response.ok(timestamp.toJSONObject().toString()).build(); + } catch (SolrServerException | IOException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); + } } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/Instances.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/Instances.java index 48375639a0..0d528c0ba1 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/Instances.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/Instances.java @@ -2,19 +2,18 @@ import java.util.List; - public interface Instances { - - public List allInstances(); - - public List enabledInstances(); - - public List disabledInstances(); - - public OneInstance find(String acronym); - - public boolean isAnyDisabled(); - - public void cronRefresh(); - + + public List allInstances(); + + public List enabledInstances(); + + public List disabledInstances(); + + public OneInstance find(String acronym); + + public boolean isAnyDisabled(); + + public void cronRefresh(); + } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java index 989204477d..e12f1dc961 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java @@ -30,106 +30,103 @@ import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance.TypeOfChangedStatus; import cz.incad.kramerius.utils.conf.KConfiguration; -public class DefaultPropertiesInstances implements Instances{ - - public static final Logger LOGGER = Logger.getLogger(DefaultPropertiesInstances.class.getName()); - - private List instances = new ArrayList<>(); - - public DefaultPropertiesInstances() { - super(); - Set names = new HashSet<>(); - Configuration configuration = KConfiguration.getInstance().getConfiguration(); - Iterator keys = configuration.getKeys("cdk.collections.sources"); - while(keys.hasNext()) { - String key = keys.next(); - if (key.contains("cdk.collections.sources.")) { - String rest = key.substring("cdk.collections.sources.".length()); - if (rest.lastIndexOf(".")>0) { - String acronym = rest.substring(0, rest.lastIndexOf(".")); - if (!names.contains(acronym)) { - names.add(acronym); - instances.add(new DefaultOnePropertiesInstance(this,acronym)); - } - } - } - } - } - - - @Override - public List allInstances() { - return this.instances; - } - - @Override - public List enabledInstances() { - return this.instances.stream().filter(OneInstance::isConnected).collect(Collectors.toList()); - } - - @Override - public List disabledInstances() { - return this.instances.stream().filter(it-> { - return !it.isConnected(); - }).collect(Collectors.toList()); - } - - @Override - public boolean isAnyDisabled() { - List eInsts = this.enabledInstances(); - return this.allInstances().size() != eInsts.size(); - } - - - - @Override - public OneInstance find(String acronym) { - - List collect = this.instances.stream().filter(it-> { - return it.getName().equals(acronym); - }).collect(Collectors.toList()); - - if (!collect.isEmpty()) { - return collect.get(0); - } else { - return null; - } - } - - - @Override - public void cronRefresh() { - try { - Map statuses = new HashMap<>(); - Client client = Client.create(); - String string = registerData(client); - JSONArray jsonArray = new JSONArray(string); - for (int i = 0,ll=jsonArray.length(); i < ll; i++) { - JSONObject oneObject = jsonArray.getJSONObject(i); - String code = oneObject.optString("code"); - Boolean status = oneObject.optBoolean("alive"); - statuses.put(code, status); - } - - for (OneInstance oneInstance : instances) { - boolean isConnected = oneInstance.isConnected(); - TypeOfChangedStatus type = oneInstance.getType(); - Boolean registrStatus = statuses.get(oneInstance.getName()); - - if (registrStatus.booleanValue() != isConnected) { - oneInstance.setConnected(registrStatus.booleanValue(), TypeOfChangedStatus.automat); - } - } - } catch (UniformInterfaceException | ClientHandlerException | JSONException | IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } - } - - - protected String registerData(Client client) throws IOException { - WebResource r = client.resource("https://api.registr.digitalniknihovna.cz/api/libraries?detail=status"); - InputStream inputStream = r.accept(MediaType.APPLICATION_XML).get(InputStream.class); - String string = org.apache.commons.io.IOUtils.toString(inputStream, "UTF-8"); - return string; - } +public class DefaultPropertiesInstances implements Instances { + + public static final Logger LOGGER = Logger.getLogger(DefaultPropertiesInstances.class.getName()); + + private List instances = new ArrayList<>(); + + public DefaultPropertiesInstances() { + super(); + Set names = new HashSet<>(); + Configuration configuration = KConfiguration.getInstance().getConfiguration(); + Iterator keys = configuration.getKeys("cdk.collections.sources"); + while (keys.hasNext()) { + String key = keys.next(); + if (key.contains("cdk.collections.sources.")) { + String rest = key.substring("cdk.collections.sources.".length()); + if (rest.lastIndexOf(".") > 0) { + String acronym = rest.substring(0, rest.lastIndexOf(".")); + if (!names.contains(acronym)) { + names.add(acronym); + instances.add(new DefaultOnePropertiesInstance(this, acronym)); + } + } + } + } + } + + @Override + public List allInstances() { + return this.instances; + } + + @Override + public List enabledInstances() { + return this.instances.stream().filter(OneInstance::isConnected).collect(Collectors.toList()); + } + + @Override + public List disabledInstances() { + return this.instances.stream().filter(it -> { + return !it.isConnected(); + }).collect(Collectors.toList()); + } + + @Override + public boolean isAnyDisabled() { + List eInsts = this.enabledInstances(); + return this.allInstances().size() != eInsts.size(); + } + + @Override + public OneInstance find(String acronym) { + List collect = this.instances.stream().filter(it -> { + return it.getName().equals(acronym); + }).collect(Collectors.toList()); + + if (!collect.isEmpty()) { + return collect.get(0); + } else { + return null; + } + } + + @Override + public void cronRefresh() { + try { + Map statuses = new HashMap<>(); + Client client = Client.create(); + String string = registerData(client); + JSONArray jsonArray = new JSONArray(string); + for (int i = 0, ll = jsonArray.length(); i < ll; i++) { + JSONObject oneObject = jsonArray.getJSONObject(i); + String code = oneObject.optString("code"); + Boolean status = oneObject.optBoolean("alive"); + statuses.put(code, status); + } + + for (OneInstance oneInstance : instances) { + boolean isConnected = oneInstance.isConnected(); + TypeOfChangedStatus type = oneInstance.getType(); + if (type.equals(TypeOfChangedStatus.user) && !isConnected) { + // musi vynechat, vypnul uzivatel + } else { + Boolean registrStatus = statuses.get(oneInstance.getName()); + if (registrStatus.booleanValue() != isConnected) { + oneInstance.setConnected(registrStatus.booleanValue(), TypeOfChangedStatus.automat); + } + } + } + } catch (UniformInterfaceException | ClientHandlerException | JSONException | IOException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } + } + + protected String registerData(Client client) throws IOException { + WebResource r = client.resource("https://api.registr.digitalniknihovna.cz/api/libraries?detail=status"); + InputStream inputStream = r.accept(MediaType.APPLICATION_XML).get(InputStream.class); + String string = org.apache.commons.io.IOUtils.toString(inputStream, "UTF-8"); + return string; + } } diff --git a/search/src/java/cz/incad/Kramerius/backend/guice/BaseModule.java b/search/src/java/cz/incad/Kramerius/backend/guice/BaseModule.java index 68685da73d..9077f194e5 100644 --- a/search/src/java/cz/incad/Kramerius/backend/guice/BaseModule.java +++ b/search/src/java/cz/incad/Kramerius/backend/guice/BaseModule.java @@ -36,8 +36,6 @@ import cz.incad.kramerius.statistics.accesslogs.dnnt.DNNTStatisticsAccessLogImpl; import cz.incad.kramerius.statistics.impl.*; import cz.incad.kramerius.utils.conf.KConfiguration; -import cz.incad.kramerius.virtualcollections.CDKVirtualCollectionsGet; -import cz.incad.kramerius.virtualcollections.impl.CDKVirtualCollectionsGetImpl; import cz.incad.kramerius.virtualcollections.Collection; import cz.incad.kramerius.virtualcollections.CollectionsManager; import cz.incad.kramerius.virtualcollections.impl.fedora.FedoraCollectionsManagerImpl; @@ -114,8 +112,6 @@ protected void configure() { lfhooks.addBinding().to(HttpAsyncClientLifeCycleHook.class); // lfhooks.addBinding().to(AudioLifeCycleHook.class); - // only CDK - bind(CDKVirtualCollectionsGet.class).to(CDKVirtualCollectionsGetImpl.class); } @Provides diff --git a/shared/common/src/main/java/cz/incad/kramerius/virtualcollections/CDKVirtualCollectionsGet.java b/shared/common/src/main/java/cz/incad/kramerius/virtualcollections/CDKVirtualCollectionsGet.java deleted file mode 100644 index 6ad034618e..0000000000 --- a/shared/common/src/main/java/cz/incad/kramerius/virtualcollections/CDKVirtualCollectionsGet.java +++ /dev/null @@ -1,14 +0,0 @@ -package cz.incad.kramerius.virtualcollections; - -import java.util.List; - -public interface CDKVirtualCollectionsGet { - - public List virtualCollectionsFromResource(String res); - - public Collection virtualCollectionsFromResource(String vc, String res); - - public List virtualCollections(); - - public String getResource(String vcId); -} diff --git a/shared/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKResourcesFilter.java b/shared/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKResourcesFilter.java deleted file mode 100644 index 0dd5ee0597..0000000000 --- a/shared/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKResourcesFilter.java +++ /dev/null @@ -1,47 +0,0 @@ -package cz.incad.kramerius.virtualcollections.impl; - -import java.util.List; -import java.util.stream.Collectors; - -import cz.incad.kramerius.utils.conf.KConfiguration; -import cz.incad.kramerius.virtualcollections.Collection; - -public class CDKResourcesFilter { - - public List getResources() { - List al = KConfiguration.getInstance().getConfiguration().getList("cdk.collections.sources").stream().map(Object::toString).collect(Collectors.toList()); - return al; - } - - public List getHidden() { - List al = KConfiguration.getInstance().getConfiguration().getList("cdk.collections.hidden").stream().map(Object::toString).collect(Collectors.toList()); - return al; - } - - public boolean isResource(Collection vc) { - return isResource(vc.getPid()); - } - - public boolean isResource(String pid) { - List al = getResources(); - return al.contains(pid); - } - - public boolean isHidden(Collection vc) { - return isHidden(vc.getPid()); - } - - public boolean isHidden(String pid) { - List al = getHidden(); - return al.contains(pid); - } - - public boolean isFiltered(Collection vc) { - return isResource(vc.getPid()) || isHidden(vc.getPid()); - } - - public boolean isFiltered(String pid) { - return isResource(pid) || isHidden(pid); - } - -} diff --git a/shared/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java b/shared/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java deleted file mode 100644 index 88ec2c740c..0000000000 --- a/shared/common/src/main/java/cz/incad/kramerius/virtualcollections/impl/CDKVirtualCollectionsGetImpl.java +++ /dev/null @@ -1,275 +0,0 @@ -package cz.incad.kramerius.virtualcollections.impl; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.logging.Level; -import java.util.logging.Logger; - -import javax.ws.rs.core.MediaType; -import javax.xml.xpath.XPathExpressionException; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; -import org.w3c.dom.Document; - -import com.google.inject.Inject; -import com.google.inject.name.Named; -import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientHandlerException; -import com.sun.jersey.api.client.UniformInterfaceException; -import com.sun.jersey.api.client.WebResource; -import com.sun.jersey.api.client.config.ClientConfig; - -import cz.incad.kramerius.FedoraAccess; -import cz.incad.kramerius.utils.DCUtils; -import cz.incad.kramerius.virtualcollections.CDKVirtualCollectionsGet; -import cz.incad.kramerius.virtualcollections.Collection; - -public class CDKVirtualCollectionsGetImpl implements CDKVirtualCollectionsGet { - - // TODO: Synchronized together -// private static Map> COLLECTIONS_MAPPING = new HashMap>(); -// private static Map COLLECTIONS_CACHE = new HashMap(); - - private static SimpleCollectionsCache _CACHE = new SimpleCollectionsCache(); - - private static final String URI_PREFIX = "URI"; - - public static final Logger LOGGER = Logger - .getLogger(CDKVirtualCollectionsGetImpl.class.getName()); - - @Inject - @Named("securedFedoraAccess") - FedoraAccess fedoraAccess; - - public CDKVirtualCollectionsGetImpl() { - } - - - public CDKVirtualCollectionsGetImpl(SimpleCollectionsCache cache) { - _CACHE = cache; - } - - public synchronized List virtualCollectionsFromResource( - String resource) { - List vcs = new ArrayList(); - CDKResourcesFilter filter = createResourceFilter(); - if (!filter.isHidden(resource)) { - try { - if (!_CACHE.containsResource(resource)) { - Document dc = this.fedoraAccess.getDC(resource); - String vcpoint = appendVCPoint(disectURL(dc)); - _CACHE.fillCacheFromResoure(resource, vcpoint); - } - JSONArray jsonArray = _CACHE.getResourceJSONArray(resource); - for (int i = 0, ll = jsonArray.length(); i < ll; i++) { - Collection vc = collectionFromJSON(jsonArray - .getJSONObject(i)); - vcs.add(vc); - } - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } catch (XPathExpressionException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } catch (JSONException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } - } - return vcs; - } - - CDKResourcesFilter createResourceFilter() { - return new CDKResourcesFilter(); - } - - public List virtualCollections() { - //clearAllResources(); - List vcs = new ArrayList(); - CDKResourcesFilter filter = createResourceFilter(); - List resources = filter.getResources(); - for (String res : resources) { - List subVCS = virtualCollectionsFromResource(res); - vcs.addAll(subVCS); - } - return vcs; - } - - @Override - public Collection virtualCollectionsFromResource(String vc, - String resource) { - CDKResourcesFilter filter = createResourceFilter(); - if (filter.isResource(resource)) { - if (!filter.isHidden(resource)) { - try { - - if (!_CACHE.containsResource(resource)) { - Document dc = this.fedoraAccess.getDC(resource); - String vcpoint = appendVCPoint(disectURL(dc)); - _CACHE.fillCacheFromResoure(resource, vcpoint); - } - JSONArray jsonArray = _CACHE.getResourceJSONArray(resource); - for (int i = 0,ll=jsonArray.length(); i < ll; i++) { - JSONObject jsonObject = jsonArray.getJSONObject(i); - Collection vcObject = collectionFromJSON(jsonObject); - if (vc.equals(vcObject.getPid())) return vcObject; - - } - - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } catch (XPathExpressionException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } catch (JSONException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } - } - } - return null; - } - - @Override - public String getResource(String vcId) { - try { - CDKResourcesFilter filter = createResourceFilter(); - List resources = filter.getResources(); - for (String res : resources) { - boolean containsResource = _CACHE.containsResource(res); - if (!containsResource) { - Document dc = this.fedoraAccess.getDC(res); - String vcpoint = appendVCPoint(disectURL(dc)); - _CACHE.fillCacheFromResoure(res, vcpoint); - } - JSONArray jsonArray = _CACHE.getResourceJSONArray(res); - for (int i = 0,ll=jsonArray.length(); i < ll; i++) { - JSONObject jsonObj = jsonArray.getJSONObject(i); - String pid = jsonObj.getString("pid"); - if (pid.equals(vcId)) return res; - } - } - } catch (XPathExpressionException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } catch (JSONException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } catch (IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } - return null; - } - - - public FedoraAccess getFedoraAccess() { - return fedoraAccess; - } - - public void setFedoraAccess(FedoraAccess fedoraAccess) { - this.fedoraAccess = fedoraAccess; - } - - static Collection collectionFromJSON(JSONObject jsonObject) - throws JSONException { - - String pid = jsonObject.getString("pid"); - String label = jsonObject.getString("label"); - boolean cleave = jsonObject.getBoolean("canLeave"); - String url = jsonObject.has("url") ? jsonObject.getString("url") : ""; - Collection col = new Collection(pid,label,url ,cleave); - JSONObject descs = jsonObject.getJSONObject("descs"); - Iterator keys = descs.keys(); - while (keys.hasNext()) { - String k = (String) keys.next(); - String string = descs.getString(k); - Collection.Description desc = new Collection.Description(k, string, string); - col.addDescription(desc); - } - return col; - } - - - static JSONArray virtualCollectionsFromPoint(String point) - throws JSONException { - try { - Client c = Client.create(); - WebResource r = c.resource(point); - r.setProperty(ClientConfig.PROPERTY_FOLLOW_REDIRECTS, new Boolean(true)); - String t = r.accept(MediaType.APPLICATION_JSON).get(String.class); - JSONArray jsonArr = new JSONArray(t); - return jsonArr; - } catch (UniformInterfaceException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - - } catch (ClientHandlerException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } - return new JSONArray(); - } - - static JSONObject virtualCollectionFromPoint(String point, String pid) - throws JSONException { - Client c = Client.create(); - WebResource r = c.resource(point + "/" + pid); - String t = r.accept(MediaType.APPLICATION_JSON).get(String.class); - JSONObject json = new JSONObject(t); - return json; - } - - static String appendVCPoint(String baseUrl) { - if (!baseUrl.endsWith("/")) { - baseUrl += "/"; - } - return baseUrl + "api/v5.0/vc"; - } - - public static String disectURL(Document dc) throws XPathExpressionException { - String src = DCUtils.sourceFromDC(dc); - return src; - } - - - static class SimpleCollectionsCache { - - private Map collectionsResults = new HashMap(); - private Map collectionObject = new HashMap(); - - - public synchronized void fillCacheFromResoure(String resourcePid, String point) { - JSONArray jsonArray = virtualCollectionsFromPoint(point); - this.collectionsResults.put(resourcePid, jsonArray); - for (int i = 0,ll=jsonArray.length(); i < ll; i++) { - JSONObject jsonObject = jsonArray.getJSONObject(i); - this.collectionObject.put(jsonObject.getString("pid"), jsonObject); - - } - } - - synchronized JSONArray virtualCollectionsFromPoint(String point) { - return CDKVirtualCollectionsGetImpl.virtualCollectionsFromPoint(point); - } - - public synchronized boolean containsResource(String resourcePid) { - return collectionsResults.containsKey(resourcePid); - } - - public synchronized boolean containsCache(String colPid) { - return this.collectionObject.containsKey(colPid); - } - - public synchronized JSONArray getResourceJSONArray(String resourcePid) { - return this.collectionsResults.get(resourcePid); - } - - public synchronized JSONObject getJSONCollection(String colPid) { - return this.collectionObject.get(colPid); - } - - public synchronized boolean isEmpty() { - return this.collectionsResults.isEmpty(); - } - } - -} From 2e127107a26eaf865a2c4757658f0782bb753c46 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 29 Dec 2022 13:22:12 +0100 Subject: [PATCH 047/235] CDK update/timestamps/refresh from registr --- search/web/WEB-INF/web.xml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/search/web/WEB-INF/web.xml b/search/web/WEB-INF/web.xml index 5323f047f0..730727efc2 100644 --- a/search/web/WEB-INF/web.xml +++ b/search/web/WEB-INF/web.xml @@ -101,6 +101,7 @@ + + + + From 9f9c0a5853f50eaa0cb46801f61d49330bbab855 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 29 Dec 2022 13:37:34 +0100 Subject: [PATCH 048/235] CDK update/timestamps/refresh from registr --- search/web/WEB-INF/web.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/search/web/WEB-INF/web.xml b/search/web/WEB-INF/web.xml index 730727efc2..7a9b04d73c 100644 --- a/search/web/WEB-INF/web.xml +++ b/search/web/WEB-INF/web.xml @@ -91,6 +91,7 @@ + + + + knav + full-crawl-knav + $iteration.url$ select + PID 7000 @@ -68,6 +73,7 @@ + PID root_pid dnnt-labels contains-dnnt-labels knav diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp.xml index c447890104..4ecf164382 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp.xml @@ -4,6 +4,10 @@ --> + + knav + full-crawl-knav + @@ -11,7 +15,7 @@ select PID - 7000 + 4000 CURSOR $iteration.user$ @@ -60,10 +64,6 @@ - mlp mlp diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_1.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_1.xml new file mode 100644 index 0000000000..b9573ab458 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_1.xml @@ -0,0 +1,90 @@ + + + + + mlp + full-crawl-mlp + + + + + + created:[2010-01-01T00:00:00.000Z TO 2015-01-01T00:00:00.000Z] + + $iteration.url$ + select + + PID + 800 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + 1 + + + + + + + select + 20 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + mlp + mlp + + + + + + + + + mlp + + + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_2.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_2.xml new file mode 100644 index 0000000000..7c693dca85 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_2.xml @@ -0,0 +1,90 @@ + + + + + mlp + full-crawl-mlp + + + + + + created:[2015-01-01T00:00:00.000Z TO 2016-01-01T00:00:00.000Z] + + $iteration.url$ + select + + PID + 800 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + 1 + + + + + + + select + 20 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + mlp + mlp + + + + + + + + + mlp + + + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_3.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_3.xml new file mode 100644 index 0000000000..739e478745 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_3.xml @@ -0,0 +1,90 @@ + + + + + mlp + full-crawl-mlp + + + + + + created:[2016-01-01T00:00:00.000Z TO 2017-01-01T00:00:00.000Z] + + $iteration.url$ + select + + PID + 800 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + 1 + + + + + + + select + 20 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + mlp + mlp + + + + + + + + + mlp + + + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_4.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_4.xml new file mode 100644 index 0000000000..8f1dec909c --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_4.xml @@ -0,0 +1,90 @@ + + + + + mlp + full-crawl-mlp + + + + + + created:[2017-01-01T00:00:00.000Z TO 2018-01-01T00:00:00.000Z] + + $iteration.url$ + select + + PID + 800 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + 1 + + + + + + + select + 20 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + mlp + mlp + + + + + + + + + mlp + + + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_5.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_5.xml new file mode 100644 index 0000000000..33fba99ef9 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_5.xml @@ -0,0 +1,90 @@ + + + + + mlp + full-crawl-mlp + + + + + + created:[2018-01-01T00:00:00.000Z TO 2019-01-01T00:00:00.000Z] + + $iteration.url$ + select + + PID + 800 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + 1 + + + + + + + select + 20 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + mlp + mlp + + + + + + + + + mlp + + + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_6.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_6.xml new file mode 100644 index 0000000000..85b4a65946 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_6.xml @@ -0,0 +1,90 @@ + + + + + mlp + full-crawl-mlp + + + + + + created:[2019-01-01T00:00:00.000Z TO 2020-01-01T00:00:00.000Z] + + $iteration.url$ + select + + PID + 800 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + 1 + + + + + + + select + 20 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + mlp + mlp + + + + + + + + + mlp + + + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_7.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_7.xml new file mode 100644 index 0000000000..4b2c07032e --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_7.xml @@ -0,0 +1,90 @@ + + + + + mlp + full-crawl-mlp + + + + + + created:[2020-01-01T00:00:00.000Z TO 2022-01-01T00:00:00.000Z] + + $iteration.url$ + select + + PID + 800 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + 1 + + + + + + + select + 20 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + mlp + mlp + + + + + + + + + mlp + + + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_8.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_8.xml new file mode 100644 index 0000000000..2e6f8e7c6f --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_8.xml @@ -0,0 +1,90 @@ + + + + + mlp + full-crawl-mlp + + + + + + created:[2022-01-01T00:00:00.000Z TO 2023-01-01T00:00:00.000Z] + + $iteration.url$ + select + + PID + 800 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + 1 + + + + + + + select + 20 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + mlp + mlp + + + + + + + + + mlp + + + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk.xml index fd17575872..7bc3589279 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk.xml @@ -33,7 +33,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnnto.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnnto.xml new file mode 100644 index 0000000000..1cc46d9cd2 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnnto.xml @@ -0,0 +1,74 @@ + + + + + + + license:dnntt + + $iteration.url$ + select + pid + 4000 + CURSOR + + + 2 + + + + + $iteration.url$ + select + 20 + + pid + + true + root.pid + pid + + $destination.url$ + select + + + cdk.collection + + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + + + + + $destination.url$/update + + + + + + + + + + mzk + mzk + mzk_dnnto + + + + + + + + mzk + mzk_dnnto + + + + + + diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnntt.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnntt.xml new file mode 100644 index 0000000000..8d1259b2c0 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnntt.xml @@ -0,0 +1,74 @@ + + + + + + + license:dnntt + + $iteration.url$ + select + pid + 4000 + CURSOR + + + 2 + + + + + $iteration.url$ + select + 20 + + pid + + true + root.pid + pid + + $destination.url$ + select + + + cdk.collection + + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + + + + + $destination.url$/update + + + + + + + + + + mzk + mzk + mzk_dnntt + + + + + + + + mzk + mzk_dnntt + + + + + + diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/nfa.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/nfa.xml new file mode 100644 index 0000000000..a9efbcfa56 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/nfa.xml @@ -0,0 +1,83 @@ + + + + nfa + full-crawl-nfa + + + + + $iteration.url$ + select + + + PID + 1000 + + CURSOR + + $iteration.user$ + $iteration.pass$ + + + 1 + + + + + + + select + 20 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + + $destination.url$/update + + + + + + + + + + nfa + nfa + + + + + + + nfa + + + + + + diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkhk.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkhk.xml index edc0e899bf..208743c5d9 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkhk.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkhk.xml @@ -5,6 +5,9 @@ + svkhk + full-crawl-svkh + @@ -21,7 +24,7 @@ - 1 + 2 @@ -71,6 +74,8 @@ + PID root_pid dnnt-labels contains-dnnt-labels + svkhk diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/utils/SOLRUtils.java b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/utils/SOLRUtils.java index 0bf14d9861..d8af4a532e 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/utils/SOLRUtils.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/utils/SOLRUtils.java @@ -36,11 +36,16 @@ public class SOLRUtils { public static final Logger LOGGER = Logger.getLogger(SOLRUtils.class.getName()); - public static Map SOLR_TYPE_NAMES = new HashMap(); + public static Map SOLR_TYPE_NAMES = new HashMap<>(); + public static Map SOLR_NAME_TYPES = new HashMap<>(); static { SOLR_TYPE_NAMES.put(String.class, "str"); SOLR_TYPE_NAMES.put(Boolean.class, "bool"); SOLR_TYPE_NAMES.put(Integer.class, "int"); + + SOLR_NAME_TYPES.put("str", String.class); + SOLR_NAME_TYPES.put("bool", Boolean.class); + SOLR_NAME_TYPES.put("int", Integer.class); } public static T value(String val, Class clz) { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java index 76761745bc..5f6c8eee57 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java @@ -127,8 +127,10 @@ public void cronRefresh() { // musi vynechat, vypnul uzivatel } else { Boolean registrStatus = statuses.get(oneInstance.getName()); - if (registrStatus.booleanValue() != isConnected) { - oneInstance.setConnected(registrStatus.booleanValue(), TypeOfChangedStatus.automat); + if (registrStatus != null) { + if (registrStatus.booleanValue() != isConnected) { + oneInstance.setConnected(registrStatus.booleanValue(), TypeOfChangedStatus.automat); + } } } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java index 4598850862..40869c8a55 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java @@ -34,6 +34,7 @@ import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; import cz.incad.kramerius.security.User; +import cz.incad.kramerius.utils.conf.KConfiguration; import cz.incad.kramerius.utils.pid.LexerException; public class V5RedirectHandler extends ProxyItemHandler { @@ -107,8 +108,16 @@ public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHan @Override public Response imageThumb(RequestMethodName method) throws ProxyHandlerException { String baseurl = baseUrl(); - String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/item/" + this.pid + "/thumb"; - return buildRedirectResponse(url); + boolean streamsThumb = KConfiguration.getInstance().getConfiguration() + .getBoolean("cdk.collections.sources." + this.source + ".thumb.streams", false); + if (streamsThumb) { + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/item/" + this.pid + "/streams/IMG_THUMB"; + return buildRedirectResponse(url); + + } else { + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/item/" + this.pid + "/thumb"; + return buildRedirectResponse(url); + } } @Override From 18d1044f6e6d8f3cffe071159d7314970f3c2635 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 3 Jan 2023 19:10:58 +0100 Subject: [PATCH 051/235] V5RedirectHandler --- .../apiNew/client/v60/redirection/item/V5RedirectHandler.java | 1 + 1 file changed, 1 insertion(+) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java index 40869c8a55..c971dd3e04 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java @@ -110,6 +110,7 @@ public Response imageThumb(RequestMethodName method) throws ProxyHandlerExceptio String baseurl = baseUrl(); boolean streamsThumb = KConfiguration.getInstance().getConfiguration() .getBoolean("cdk.collections.sources." + this.source + ".thumb.streams", false); + LOGGER.info("cdk.collections.sources."+ this.source + ".thumb.streams = "+streamsThumb); if (streamsThumb) { String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/item/" + this.pid + "/streams/IMG_THUMB"; return buildRedirectResponse(url); From 2856ecc40eee1234e4e6cd5ca6c423aef068c1bb Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 3 Jan 2023 19:34:30 +0100 Subject: [PATCH 052/235] Changed property --- .../apiNew/client/v60/redirection/item/V5RedirectHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java index c971dd3e04..475203571c 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java @@ -109,8 +109,8 @@ public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHan public Response imageThumb(RequestMethodName method) throws ProxyHandlerException { String baseurl = baseUrl(); boolean streamsThumb = KConfiguration.getInstance().getConfiguration() - .getBoolean("cdk.collections.sources." + this.source + ".thumb.streams", false); - LOGGER.info("cdk.collections.sources."+ this.source + ".thumb.streams = "+streamsThumb); + .getBoolean("cdk.collections.sources." + this.source + ".thumb_streams", false); + LOGGER.info("cdk.collections.sources."+ this.source + ".thumb_streams = "+streamsThumb); if (streamsThumb) { String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/item/" + this.pid + "/streams/IMG_THUMB"; return buildRedirectResponse(url); From 2a0549ea5288586f859ef92bf9182082da4fcc43 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 3 Jan 2023 19:53:25 +0100 Subject: [PATCH 053/235] Changed version --- build.gradle | 2 +- shared/common/src/main/java/res/configuration.properties | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/build.gradle b/build.gradle index 12b249ee25..ae8667077f 100644 --- a/build.gradle +++ b/build.gradle @@ -66,7 +66,7 @@ repositories { allprojects { group='cz.incad.kramerius' - version='7.0.2-beta2_cdk' + version='7.0.3-beta2_cdk' } diff --git a/shared/common/src/main/java/res/configuration.properties b/shared/common/src/main/java/res/configuration.properties index d226ea32cd..c8ad841889 100644 --- a/shared/common/src/main/java/res/configuration.properties +++ b/shared/common/src/main/java/res/configuration.properties @@ -1,6 +1,5 @@ -#Global properties for kramerius 4 - - +#Global properties for kramerius cdk +# Version = 7.0.3-beta2_cdk ## Use this if you want to use fedora 4 instance fedora4Host=http://localhost:18080 From 40a47dbff61ee3de1ef982357566d531e1e9f7d5 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 6 Jan 2023 11:23:23 +0100 Subject: [PATCH 054/235] Update definitions --- .../configurations/inovatika.xml | 4 +- .../periodicupdates/configurations/knav.xml | 9 +- .../periodicupdates/configurations/mlp.xml | 85 ++++++++++++++++++ .../periodicupdates/configurations/nfa.xml | 81 +++++++++++++++++ .../periodicupdates/configurations/ntk.xml | 80 +++++++++++++++++ .../periodicupdates/configurations/svkul.xml | 11 ++- .../periodicupdates/configurations/uzei.xml | 87 +++++++++++++++++++ .../configurations/mzk_license_dnnto.xml | 6 +- .../configurations/mzk_license_dnntt.xml | 6 +- .../configurations/mzk_sliceed_1.xml | 6 +- .../configurations/mzk_sliceed_10.xml | 2 +- .../configurations/mzk_sliceed_11.xml | 2 +- .../configurations/mzk_sliceed_12.xml | 2 +- .../configurations/mzk_sliceed_2.xml | 2 +- .../configurations/mzk_sliceed_3.xml | 2 +- .../configurations/mzk_sliceed_4.xml | 2 +- .../configurations/mzk_sliceed_5.xml | 2 +- .../configurations/mzk_sliceed_6.xml | 2 +- .../configurations/mzk_sliceed_7.xml | 3 +- .../configurations/mzk_sliceed_8.xml | 2 +- .../configurations/mzk_sliceed_9.xml | 2 +- 21 files changed, 362 insertions(+), 36 deletions(-) create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/mlp.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/ntk.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/inovatika.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/inovatika.xml index 034e68af37..563f0acec3 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/inovatika.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/inovatika.xml @@ -1,8 +1,6 @@ - - - + inovatika inovatika-update http://localhost:8080/search/api/admin/v7.0/connected/knav/timestamp diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav.xml index 95563b28f5..6ab6baffe4 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav.xml @@ -1,12 +1,9 @@ - - + + knav knav-update - http://localhost:8080/search/api/admin/v7.0/connected/knav/timestamp + http://localhost:8080/search/api/admin/v7.0/connected/ 1 diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/mlp.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/mlp.xml new file mode 100644 index 0000000000..227bc9c779 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/mlp.xml @@ -0,0 +1,85 @@ + + + mlp + mlp-update + + http://localhost:8080/search/api/admin/v7.0/connected/ + 1 + + + + + $iteration.url$ + select + + PID + 4000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + 1 + + + + + + + select + 20 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + mlp + mlp + + + + + + + + + mlp + + + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa.xml new file mode 100644 index 0000000000..7cb0fba1a3 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa.xml @@ -0,0 +1,81 @@ + + + nfa + nfa-update + http://localhost:8080/search/api/admin/v7.0/connected/ + 1 + + + + + $iteration.url$ + select + + + PID + 1000 + + CURSOR + + $iteration.user$ + $iteration.pass$ + + + 1 + + + + + + + select + 20 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + + $destination.url$/update + + + + + + + + + + nfa + nfa + + + + + + + nfa + + + + + + diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/ntk.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/ntk.xml new file mode 100644 index 0000000000..3338ac0622 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/ntk.xml @@ -0,0 +1,80 @@ + + + ntk + ntk-update + http://localhost:8080/search/api/admin/v7.0/connected/ + 1 + + + + + $iteration.url$ + select + + PID + 7000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + 1 + + + + + + + select + 20 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + ntk + ntk + + + + + + + + ntk + + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul.xml index 8106a2eb6b..52d2c83aa4 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul.xml @@ -1,10 +1,9 @@ - - - svkul-update + + skvul + skvul-update + http://localhost:8080/search/api/admin/v7.0/connected/ + 1 http://localhost:8080/search/api/admin/v7.0/connected/svkul/timestamp 1 diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei.xml new file mode 100644 index 0000000000..26de9cc673 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei.xml @@ -0,0 +1,87 @@ + + + uzei + uzei-update + http://localhost:8080/search/api/admin/v7.0/connected/ + 1 + + + + + $iteration.url$ + select + + PID + 7000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + 1 + + + + + + + select + 20 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + + uzei + uzei + + + + + + + + + uzei + + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnnto.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnnto.xml index 1cc46d9cd2..e76c46653b 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnnto.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnnto.xml @@ -6,7 +6,7 @@ - license:dnntt + licenses:dnnto $iteration.url$ select @@ -36,7 +36,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors @@ -54,6 +54,7 @@ mzk mzk mzk_dnnto + dnnto @@ -66,6 +67,7 @@ --> mzk mzk_dnnto + dnnto diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnntt.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnntt.xml index 8d1259b2c0..953c2075ed 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnntt.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnntt.xml @@ -6,7 +6,7 @@ - license:dnntt + licenses:dnntt $iteration.url$ select @@ -36,7 +36,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors @@ -54,6 +54,7 @@ mzk mzk mzk_dnntt + dnntt @@ -66,6 +67,7 @@ --> mzk mzk_dnntt + dnntt diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_1.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_1.xml index 62e1d60fae..f890260c49 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_1.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_1.xml @@ -36,7 +36,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors @@ -51,10 +51,6 @@ - mzk mzk diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_10.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_10.xml index 3a74c29cde..0c6bdac66b 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_10.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_10.xml @@ -39,7 +39,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_11.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_11.xml index 1e38287c68..62960af9fd 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_11.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_11.xml @@ -36,7 +36,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_12.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_12.xml index 3429258b55..c2ebcf1f0a 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_12.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_12.xml @@ -37,7 +37,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_2.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_2.xml index d31579cf5f..3058ac1f95 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_2.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_2.xml @@ -36,7 +36,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_3.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_3.xml index d3d7025b13..2917cbdb6a 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_3.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_3.xml @@ -36,7 +36,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_4.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_4.xml index 12af8a973f..c733932d57 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_4.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_4.xml @@ -37,7 +37,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_5.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_5.xml index 3ab187a88b..62a4bfdefd 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_5.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_5.xml @@ -36,7 +36,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_6.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_6.xml index 13daf0d298..13727657ba 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_6.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_6.xml @@ -36,7 +36,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_7.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_7.xml index 1bb3d26503..25d6c3936a 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_7.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_7.xml @@ -36,8 +36,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr - + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_8.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_8.xml index 9090a03376..9e825743fd 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_8.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_8.xml @@ -36,7 +36,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_9.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_9.xml index 99b4a87904..b444412a9c 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_9.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_9.xml @@ -36,7 +36,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors From 1168b51d168ddf4070e140011398b79bcdc9206f Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 9 Jan 2023 10:31:17 +0100 Subject: [PATCH 055/235] CDK switch/Prepare --- .../services/KubernetesProcessImpl.java | 10 +++ .../transform/BasicSourceToDestTransform.java | 2 + .../replicate/AbstractReplicateWorker.java | 37 ++++++--- .../workers/replicate/BatchUtils.java | 46 ++++++++++- .../workers/replicate/ReplicateContext.java | 29 +++++-- .../workers/replicate/ReplicateFinisher.java | 14 ++++ .../replicate/copy/CopyReplicateConsumer.java | 2 +- .../replicate/copy/CopyReplicateWorker.java | 70 ++++++++++++---- .../updatefield/UpdateFieldWorker.java | 10 +-- .../{inovatika.xml => inovatika-update.xml} | 9 ++- .../{mlp.xml => kkp-update.xml} | 38 +++++---- .../configurations/knav-update.xml | 79 ++++++++++++++++++ .../periodicupdates/configurations/knav.xml | 81 ------------------- .../configurations/mzk-update.xml} | 23 +++--- .../{nfa.xml => nfa-update.xml} | 9 ++- .../configurations/svkhk-update.xml | 80 ++++++++++++++++++ .../{ntk.xml => svkul-update.xml} | 17 ++-- .../periodicupdates/configurations/svkul.xml | 81 ------------------- .../{uzei.xml => uzei-update.xml} | 7 +- .../replicate/configurations/inovatika.xml | 13 +-- .../workers/replicate/configurations/kkp.xml | 17 +++- .../workers/replicate/configurations/knav.xml | 23 +++--- .../workers/replicate/configurations/mlp.xml | 10 +-- .../workers/replicate/configurations/mzk.xml | 21 +++-- .../configurations/mzk_license_dnnto.xml | 76 ----------------- .../workers/replicate/configurations/nfa.xml | 13 +-- .../{ => sliced/mlp}/mlp_sliced_1.xml | 0 .../{ => sliced/mlp}/mlp_sliced_2.xml | 0 .../{ => sliced/mlp}/mlp_sliced_3.xml | 0 .../{ => sliced/mlp}/mlp_sliced_4.xml | 0 .../{ => sliced/mlp}/mlp_sliced_5.xml | 0 .../{ => sliced/mlp}/mlp_sliced_6.xml | 0 .../{ => sliced/mlp}/mlp_sliced_7.xml | 0 .../{ => sliced/mlp}/mlp_sliced_8.xml | 0 .../{ => sliced/mzk}/mzk_sliceed_1.xml | 0 .../{ => sliced/mzk}/mzk_sliceed_10.xml | 0 .../{ => sliced/mzk}/mzk_sliceed_11.xml | 0 .../{ => sliced/mzk}/mzk_sliceed_12.xml | 0 .../{ => sliced/mzk}/mzk_sliceed_2.xml | 0 .../{ => sliced/mzk}/mzk_sliceed_3.xml | 0 .../{ => sliced/mzk}/mzk_sliceed_4.xml | 0 .../{ => sliced/mzk}/mzk_sliceed_5.xml | 0 .../{ => sliced/mzk}/mzk_sliceed_6.xml | 0 .../{ => sliced/mzk}/mzk_sliceed_7.xml | 0 .../{ => sliced/mzk}/mzk_sliceed_8.xml | 0 .../{ => sliced/mzk}/mzk_sliceed_9.xml | 0 .../replicate/configurations/svkhk.xml | 10 +-- .../replicate/configurations/svkul.xml | 10 ++- .../workers/replicate/configurations/uzei.xml | 12 +-- search/web/WEB-INF/web.xml | 4 - 50 files changed, 464 insertions(+), 389 deletions(-) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/{inovatika.xml => inovatika-update.xml} (90%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/{mlp.xml => kkp-update.xml} (76%) create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav-update.xml delete mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav.xml rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/{replicate/configurations/mzk_license_dnntt.xml => periodicupdates/configurations/mzk-update.xml} (78%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/{nfa.xml => nfa-update.xml} (94%) create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkhk-update.xml rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/{ntk.xml => svkul-update.xml} (85%) delete mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul.xml rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/{uzei.xml => uzei-update.xml} (94%) delete mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnnto.xml rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mlp}/mlp_sliced_1.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mlp}/mlp_sliced_2.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mlp}/mlp_sliced_3.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mlp}/mlp_sliced_4.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mlp}/mlp_sliced_5.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mlp}/mlp_sliced_6.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mlp}/mlp_sliced_7.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mlp}/mlp_sliced_8.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mzk}/mzk_sliceed_1.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mzk}/mzk_sliceed_10.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mzk}/mzk_sliceed_11.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mzk}/mzk_sliceed_12.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mzk}/mzk_sliceed_2.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mzk}/mzk_sliceed_3.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mzk}/mzk_sliceed_4.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mzk}/mzk_sliceed_5.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mzk}/mzk_sliceed_6.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mzk}/mzk_sliceed_7.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mzk}/mzk_sliceed_8.xml (100%) rename processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/{ => sliced/mzk}/mzk_sliceed_9.xml (100%) diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java index 2d25e627ef..37415e9af3 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java @@ -22,6 +22,7 @@ */ public class KubernetesProcessImpl { + public static final String TMSP_URL ="TIMESTAMP_URL"; public static final String ITERATION_URL = "ITERATION_URL"; public static final String CHECK_URL = "CHECK_URL"; public static final String DESTINATION_URL = "DESTINATION_URL"; @@ -32,6 +33,7 @@ public class KubernetesProcessImpl { public static final String CHECK_PREFIX = "CHECK_"; public static final String DEST_PREFIX = "DESTINATION_"; + public static final String TMSP_PREFIX = "TIMESTAMP_"; public static final Logger LOGGER = Logger.getLogger(KubernetesProcessImpl.class.getName()); @@ -43,6 +45,7 @@ public static void main(String[] args) throws IOException, MigrateSolrIndexExcep InputStream stream = KubernetesProcessImpl.class.getResourceAsStream(configSource); if (stream != null) { + Map iteration = new HashMap<>(); prefixVariables(env, iteration, ITERATION_PREFIX); if (env.containsKey(ITERATION_URL)) { @@ -61,12 +64,19 @@ public static void main(String[] args) throws IOException, MigrateSolrIndexExcep destination.put("url", env.get(DESTINATION_URL)); } + Map timestamps = new HashMap<>(); + prefixVariables(env, timestamps, TMSP_PREFIX); + if (env.containsKey(TMSP_PREFIX)) { + destination.put("url", env.get(DESTINATION_URL)); + } + StringTemplate template = new StringTemplate( IOUtils.toString(stream, "UTF-8"), DefaultTemplateLexer.class); template.setAttribute("iteration", iteration); template.setAttribute("check", check); template.setAttribute("destination", destination); + template.setAttribute("timestamp", timestamps); String configuration = template.toString(); LOGGER.info("Loading configuration "+configuration); diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/BasicSourceToDestTransform.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/BasicSourceToDestTransform.java index 8aa11fd8f3..bda343dae2 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/BasicSourceToDestTransform.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/BasicSourceToDestTransform.java @@ -17,6 +17,8 @@ import java.util.function.Consumer; import java.util.logging.Level; + +//TODO: Change it in future; copied from K5 cdk download public class BasicSourceToDestTransform extends SourceToDestTransform{ // text is copied but not for PDF; uuugrrrr !!! Terrible diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java index 258f5f5e40..f21d18de9a 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java @@ -41,6 +41,7 @@ public abstract class AbstractReplicateWorker extends Worker { protected String idIdentifier = DEFAULT_PID_FIELD; protected String collectionField = COLLECTION_FIELD; + protected boolean compositeId = false; protected String rootOfComposite = null; protected String childOfComposite = null; @@ -52,6 +53,7 @@ public AbstractReplicateWorker(String sourceName, Element workerElm, Client clie super(sourceName, workerElm, client, items); } + // zjistuje, ziskava vsechny pidy, ktere jsou naindexovane protected ReplicateContext findPidsAlreadyIndexed(List subpids, SourceToDestTransform transform) throws ParserConfigurationException, SAXException, IOException { @@ -61,15 +63,16 @@ protected ReplicateContext findPidsAlreadyIndexed(List subpids, SourceTo }).collect(Collectors.joining(" OR ")); // field list je pid + collection + rootpid + dalsi field list - String fieldlist = this.transform.getField(idIdentifier) + " " + collectionField +" cdk.leader"; + //String fieldlist = this.transform.getField(idIdentifier) + " " + collectionField +" cdk.leader cdk.licenses cdk.collection"; + String fieldlist = "pid " + collectionField +" cdk.leader cdk.licenses cdk.collection"; if (compositeId) { - fieldlist = fieldlist + " " + this.transform.getField(this.rootOfComposite); - if (!idIdentifier.equals(childOfComposite)) { - fieldlist = fieldlist + " " + this.transform.getField(this.childOfComposite); - } + fieldlist = fieldlist + " " + " root.pid compositeId"; +// if (!idIdentifier.equals(childOfComposite)) { +// fieldlist = fieldlist + " " + this.transform.getField(this.childOfComposite); +// } } - String query = "?q=" + this.transform.getField(idIdentifier) + ":(" + URLEncoder.encode(reduce, "UTF-8") + String query = "?q=" + "pid" + ":(" + URLEncoder.encode(reduce, "UTF-8") + ")&fl=" + URLEncoder.encode(fieldlist, "UTF-8") + "&wt=xml&rows=" + subpids.size(); String checkUrl = this.checkUrl + (this.checkUrl.endsWith("/") ? "" : "/") + this.checkEndpoint; @@ -79,16 +82,17 @@ protected ReplicateContext findPidsAlreadyIndexed(List subpids, SourceTo }); List docs = XMLUtils.getElements(resultElem); - List> list = new ArrayList<>(); + List> list = new ArrayList<>(); docs.stream().forEach(d -> { - Map map = new HashMap<>(); + Map map = new HashMap<>(); Element pid = XMLUtils.findElement(d, e -> { - return e.getAttribute("name").equals(this.transform.getField(idIdentifier)); + return e.getAttribute("name").equals("pid"); }); if (pid != null) { - map.put(this.transform.getField(idIdentifier), pid.getTextContent()); + map.put("pid", pid.getTextContent()); } + Element collection = XMLUtils.findElement(d, e -> { return e.getAttribute("name").equals(collectionField); }); @@ -105,6 +109,17 @@ protected ReplicateContext findPidsAlreadyIndexed(List subpids, SourceTo map.put("cdk.leader", cdkLeader.getTextContent()); } + Element cdkLicenses = XMLUtils.findElement(d, e -> { + return e.getAttribute("name").equals("cdk.licenses"); + }); + + if (cdkLicenses != null) { + List licenses = XMLUtils.getElements(cdkLicenses).stream().map(Element::getTextContent).collect(Collectors.toList()); + map.put("cdk.licenses", licenses); + } + + + if (compositeId) { Element compositeRoot = XMLUtils.findElement(d, e -> { return e.getAttribute("name").equals(this.transform.getField(rootOfComposite)); @@ -125,7 +140,7 @@ protected ReplicateContext findPidsAlreadyIndexed(List subpids, SourceTo }); List pidsFromLocalSolr = list.stream().map(m -> { - return m.get(this.transform.getField(idIdentifier)); + return m.get("pid").toString(); }).collect(Collectors.toList()); List notindexed = new ArrayList<>(); diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/BatchUtils.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/BatchUtils.java index 7d805cd4ca..cefb1e7a7e 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/BatchUtils.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/BatchUtils.java @@ -37,6 +37,15 @@ public boolean acceptElement(Element elm) { } }); + StringWriter writer = new StringWriter(); + try { + XMLUtils.print(resultElem, writer); + } catch (TransformerException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + for (int i = 0; i < docs.size(); i++) { Element destDocElement = destBatch.createElement("doc"); Element sourceDocElm = docs.get(i); @@ -44,6 +53,15 @@ public boolean acceptElement(Element elm) { // basic transform srcTransform.transform(sourceDocElm, destBatch, destDocElement,consumer); + writer = new StringWriter(); + try { + XMLUtils.print(destDocElement, writer); + } catch (TransformerException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + // composite id is not supported if (compositeId && root != null && child != null) { @@ -61,8 +79,34 @@ public boolean acceptElement(Element elm) { destBatch.getDocumentElement().appendChild(destDocElement); } - consumer.changeDocument(destDocElement); + Element rootPidElm = XMLUtils.findElement(destDocElement, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + return name.equals("root.pid"); + } + }); + + + Element pidElm = XMLUtils.findElement(destDocElement, new XMLUtils.ElementsFilter() { + + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + return name.equals("pid"); + } + }); + + writer = new StringWriter(); + try { + XMLUtils.print(destDocElement, writer); + } catch (TransformerException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + consumer.changeDocument(rootPidElm.getTextContent(),pidElm.getTextContent(), destDocElement); } return destBatch; } diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateContext.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateContext.java index ec15f1244c..6b2ea1a52e 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateContext.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateContext.java @@ -1,28 +1,43 @@ package cz.incad.kramerius.services.workers.replicate; -import org.apache.commons.lang3.tuple.Pair; +import java.util.HashMap; import java.util.List; import java.util.Map; /** - * Represents object which contains information what should be replicated + * Rerpresents the replication context */ public class ReplicateContext { - - private List> alreadyIndexed; - + + // Already indexed documents: + // retrieved information: + // - pid + // - cdk.leader + // - composeId + private List> alreadyIndexed; + + // Not indexed documents - pids private List notIndexed; - public ReplicateContext(List> alreadyIndexed, List notIndexed) { + public ReplicateContext(List> alreadyIndexed, List notIndexed) { this.alreadyIndexed = alreadyIndexed; this.notIndexed = notIndexed; } - public List> getAlreadyIndexed() { + public List> getAlreadyIndexed() { return alreadyIndexed; } + public Map> getAlreadyIndexedAsMap() { + Map> map = new HashMap<>(); + this.alreadyIndexed.stream().forEach(m-> { + String pid = m.get("pid").toString(); + map.put(pid, m); + }); + return map; + } + public List getNotIndexed() { return notIndexed; } diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateFinisher.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateFinisher.java index aa3b83cf69..794022d99c 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateFinisher.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateFinisher.java @@ -7,6 +7,7 @@ import cz.incad.kramerius.services.utils.SolrUtils; import cz.incad.kramerius.timestamps.TimestampStore; import cz.incad.kramerius.utils.StringUtils; +import cz.incad.kramerius.utils.XMLUtils; import org.json.JSONObject; import org.w3c.dom.Element; @@ -39,8 +40,17 @@ public class ReplicateFinisher extends WorkerFinisher { long start = System.currentTimeMillis(); + protected String typeOfCrawl; + + public ReplicateFinisher(String timestampUrl, Element workerElm, Client client) { super(timestampUrl, workerElm, client); + + Element typeElm = XMLUtils.findElement((Element)workerElm.getParentNode(), "type"); + if (typeElm != null) { + typeOfCrawl = typeElm.getTextContent(); + } + } private JSONObject storeTimestamp() { @@ -50,6 +60,10 @@ private JSONObject storeTimestamp() { jsonObject.put("indexed", NEWINDEXED); jsonObject.put("updated", UPDATED); + if (typeOfCrawl != null) { + jsonObject.put("type", typeOfCrawl); + } + LOGGER.info(String.format("[" + Thread.currentThread().getName() + "] url %s", timestampUrl)); WebResource r = client.resource(timestampUrl); String t = r.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).entity(jsonObject.toString()).put(String.class); diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateConsumer.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateConsumer.java index b5e02cfb9a..dc38fe573b 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateConsumer.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateConsumer.java @@ -10,5 +10,5 @@ public enum ModifyFieldResult { public ModifyFieldResult modifyField(Element field); - public void changeDocument(Element doc); + public void changeDocument(String rootPid, String pid, Element doc); } diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java index 5332771b0b..7dcb23fc24 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java @@ -19,9 +19,11 @@ import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.concurrent.BrokenBarrierException; import java.util.function.Consumer; import java.util.logging.Level; @@ -70,7 +72,7 @@ public ModifyFieldResult modifyField(Element field) { @Override - public void changeDocument(Element doc) { + public void changeDocument(String rootPid, String pid,Element doc) { List foundElements = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { @Override @@ -87,7 +89,6 @@ public boolean acceptElement(Element element) { cdkLicenses.setAttribute("name", "cdk.licenses"); cdkLicenses.setTextContent(sourceName+"_"+ fElm.getTextContent()); - //cdkLicenses.setAttribute("update", "add-distinct"); doc.appendChild(cdkLicenses); } } @@ -122,23 +123,31 @@ public boolean acceptElement(Element element) { LOGGER.info(s); } - // + // Jiz indexovane pidy if (!pidsToReplicate.getAlreadyIndexed().isEmpty()) { - // un update + // un update udalost if (!this.onUpdateUpdateElements.isEmpty()) { - + // pokud je definovany fieldlist pak musime ziskat data ze zdrojoveho krameria String fl = this.onUpdateFieldList != null ? this.onUpdateFieldList : null; Document destBatch = null; if (fl != null) { + // indexovane pidy & a mapovani pid <=> dokument List pids = pidsToReplicate.getAlreadyIndexed().stream().map(pair->{ - String string = pair.get(transform.getField(childOfComposite)); + String string = pair.get("pid").toString(); return string; }).collect(Collectors.toList()); - + Map> docs = pidsToReplicate.getAlreadyIndexedAsMap(); + destBatch = retrieveAndCretebatch(pids, fl, new CopyReplicateConsumer() { + /** + * Modifikovani pole + * @param field + * @return + */ @Override public ModifyFieldResult modifyField(Element field) { + List deleteFields = Arrays.asList(); List addValues = Arrays.asList("licenses","licenses_of_ancestors","contains_licenses"); @@ -147,7 +156,7 @@ public ModifyFieldResult modifyField(Element field) { return ModifyFieldResult.delete; } else { - //licenses_of_ancestors + // pridavani poli if (addValues.contains(name)) { field.setAttribute("update", "add-distinct"); } else { @@ -159,8 +168,12 @@ public ModifyFieldResult modifyField(Element field) { } @Override - public void changeDocument(Element doc) { - List foundElements = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { + public void changeDocument(String root, String pid, Element doc) { + + // cdk dokument + Map cdkDoc = docs.get(pid); + + List newIndexedLicenses = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { @Override public boolean acceptElement(Element element) { @@ -169,8 +182,22 @@ public boolean acceptElement(Element element) { } }); - if (!foundElements.isEmpty()) { - for (Element fElm : foundElements) { + + List newCDKLicenses = new ArrayList<>(); + newCDKLicenses = newIndexedLicenses.stream().map(Element::getTextContent).map(lic-> { + return sourceName+"_"+lic; + }).collect(Collectors.toList()); + + List indexedCDKLicenses = (List) cdkDoc.get("cdk.licenses"); + if (indexedCDKLicenses == null) { + indexedCDKLicenses = new ArrayList<>(); + } + + List toRemoveCDKLicenses = new ArrayList<>(indexedCDKLicenses); + toRemoveCDKLicenses.removeAll(newCDKLicenses); + + if (!newCDKLicenses.isEmpty()) { + for (Element fElm : newIndexedLicenses) { Document document = fElm.getOwnerDocument(); Element cdkLicenses = document.createElement("field"); cdkLicenses.setAttribute("name", "cdk.licenses"); @@ -181,6 +208,17 @@ public boolean acceptElement(Element element) { } } + if (toRemoveCDKLicenses!=null && toRemoveCDKLicenses.size() > 0) { + for (String remove : toRemoveCDKLicenses) { + Document document = doc.getOwnerDocument(); + Element cdkLicenses = document.createElement("field"); + cdkLicenses.setAttribute("name", "cdk.licenses"); + cdkLicenses.setTextContent(sourceName+"_"+remove); + cdkLicenses.setAttribute("update", "remove"); + doc.appendChild(cdkLicenses); + } + } + } }); } else { @@ -189,10 +227,10 @@ public boolean acceptElement(Element element) { Element doc = db.createElement("doc"); Element field = db.createElement("field"); if (compositeId) { - String compositeId = pair.get("compositeId"); + String compositeId = pair.get("compositeId").toString(); - String root = pair.get(transform.getField(rootOfComposite)); - String child = pair.get(transform.getField(childOfComposite)); + String root = pair.get(transform.getField(rootOfComposite)).toString(); + String child = pair.get(transform.getField(childOfComposite)).toString(); field.setAttribute("name", "compositeId"); field.setTextContent(root +"!"+child); @@ -200,7 +238,7 @@ public boolean acceptElement(Element element) { } else { String idname = transform.getField(idIdentifier); - String identifier = pair.get(idname); + String identifier = pair.get(idname).toString(); // if compositeid field.setAttribute("name", idname); // formal name from hashmap diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/updatefield/UpdateFieldWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/updatefield/UpdateFieldWorker.java index 031c233952..a5a2ccc656 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/updatefield/UpdateFieldWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/updatefield/UpdateFieldWorker.java @@ -69,7 +69,7 @@ public void run() { if (!pidsToReplicate.getAlreadyIndexed().isEmpty()) { List pids = pidsToReplicate.getAlreadyIndexed().stream().map(m -> { - return m.get(this.idIdentifier); + return m.get(this.idIdentifier).toString(); }).collect(Collectors.toList()); Element response = fetchFields(pids); @@ -116,15 +116,15 @@ public void run() { if (compositeId) { // String compositeId = pair.get("compositeId"); - String root = pair.get(transform.getField(rootOfComposite)); - String child = pair.get(transform.getField(childOfComposite)); + String root = pair.get(transform.getField(rootOfComposite)).toString(); + String child = pair.get(transform.getField(childOfComposite)).toString(); field.setAttribute("name", "compositeId"); field.setTextContent(root + "!" + child); } else { String idname = transform.getField(idIdentifier); - String identifier = pair.get(idname); + String identifier = pair.get(idname).toString(); // if compositeid field.setAttribute("name", idname); // formal name from hashmap @@ -133,7 +133,7 @@ public void run() { doc.appendChild(field); String idname = transform.getField(idIdentifier); - String identifier = pair.get(idname); + String identifier = pair.get(idname).toString(); if (map.containsKey(identifier)) { List values = map.get(identifier).getLeft(); for (String value : values) { diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/inovatika.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/inovatika-update.xml similarity index 90% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/inovatika.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/inovatika-update.xml index 563f0acec3..3ff3b1d0f1 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/inovatika.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/inovatika-update.xml @@ -1,9 +1,10 @@ - inovatika + inovatika inovatika-update + update - http://localhost:8080/search/api/admin/v7.0/connected/knav/timestamp + http://localhost:8080/search/api/admin/v7.0/connected/ 1 @@ -50,7 +51,7 @@ cdk.collection - pid root.* own_parent.* own_pid_path own_model_path + pid root.* own_parent.* own_pid_path own_model_path licenses licenses_of_ancestors contains_licenses @@ -73,6 +74,8 @@ + pid root.* own_parent.* own_pid_path own_model_path + inovatika diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/mlp.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/kkp-update.xml similarity index 76% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/mlp.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/kkp-update.xml index 227bc9c779..ce69d08d3a 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/mlp.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/kkp-update.xml @@ -1,10 +1,13 @@ - - - mlp - mlp-update - - http://localhost:8080/search/api/admin/v7.0/connected/ - 1 + + + + kkp + kkp-update + update + + $timestamp.url$ @@ -13,11 +16,15 @@ select PID - 4000 + 5000 CURSOR $iteration.user$ $iteration.pass$ + + + modified_date + 1 @@ -62,20 +69,19 @@ - mlp - mlp + kkp + kkp - - - mlp - + + PID root_pid dnnt-labels contains-dnnt-labels + + + kkp diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav-update.xml new file mode 100644 index 0000000000..148e79f51e --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav-update.xml @@ -0,0 +1,79 @@ + + + knav + knav-update + update + + $timestamp.url$ + 1 + + + + + $iteration.url$ + search + PID + CURSOR + + + modified_date + + root_pid:"uuid:477085e4-50f6-4893-a639-229a41fc7a3f" + + + + + + + $iteration.url$ + search + 20 + + PID + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + knav + knav + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels + + + knav + knav + + + + + + diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav.xml deleted file mode 100644 index 6ab6baffe4..0000000000 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav.xml +++ /dev/null @@ -1,81 +0,0 @@ - - - knav - knav-update - - http://localhost:8080/search/api/admin/v7.0/connected/ - 1 - - - - - inovatika - $iteration.url$ - select - PID - CURSOR - - - modified_date - - root_pid:"uuid:1000fca9-a34d-4891-bb25-50db86fb3253" - - - - - - - - $iteration.url$ - select - 20 - - PID - - true - root.pid - pid - - $destination.url$ - select - - - cdk.collection - - - - PID root_pid dc.title title document_type - - - - - - - $destination.url$/update - - - - - - - - - - inovatika - inovatika - - - - - - - inovatika - inovatika - - - - - - diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnntt.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/mzk-update.xml similarity index 78% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnntt.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/mzk-update.xml index 953c2075ed..c73c7e455a 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnntt.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/mzk-update.xml @@ -1,13 +1,14 @@ - + + + mzk + mzk-update + + $timestamp.url$ + - - licenses:dnntt - $iteration.url$ select pid @@ -53,21 +54,15 @@ mzk mzk - mzk_dnntt - dnntt + pid root.pid licenses contains_licenses licenses_of_ancestors + - mzk - mzk_dnntt - dnntt diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa-update.xml similarity index 94% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa-update.xml index 7cb0fba1a3..212df2e4fb 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa-update.xml @@ -1,8 +1,12 @@ - nfa + nfa nfa-update - http://localhost:8080/search/api/admin/v7.0/connected/ + update + + $timestamp.url$ + + 1 @@ -50,7 +54,6 @@ K7 - diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkhk-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkhk-update.xml new file mode 100644 index 0000000000..a3ae4b90b5 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkhk-update.xml @@ -0,0 +1,80 @@ + + + + svkhk + svkh-full + + $timestamp.url$ + + + + + $iteration.url$ + select + + PID + 7000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + + + 2 + + + + + + + select + 20 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + $destination.url$/update + + + + + + + + + + svkhk + svkhk + + + + + PID root_pid dnnt-labels contains-dnnt-labels + + + svkhk + + + + + + diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/ntk.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul-update.xml similarity index 85% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/ntk.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul-update.xml index 3338ac0622..8d9e985dd9 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/ntk.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul-update.xml @@ -1,9 +1,10 @@ + - ntk - ntk-update - http://localhost:8080/search/api/admin/v7.0/connected/ - 1 + svkul + svkul-full + + $timestamp.url$ @@ -12,7 +13,7 @@ select PID - 7000 + 5000 CURSOR $iteration.user$ @@ -61,8 +62,8 @@ - ntk - ntk + svkul + svkul @@ -70,7 +71,7 @@ - ntk + svkul diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul.xml deleted file mode 100644 index 52d2c83aa4..0000000000 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul.xml +++ /dev/null @@ -1,81 +0,0 @@ - - - skvul - skvul-update - http://localhost:8080/search/api/admin/v7.0/connected/ - 1 - - http://localhost:8080/search/api/admin/v7.0/connected/svkul/timestamp - 1 - - - - - inovatika - $iteration.url$ - select - PID - CURSOR - - - modified_date - - - - - - $iteration.url$ - select - 20 - - - true - root_pid - PID - - - $destination.url$ - select - - - cdk.collection - - - - PID root_pid dc.title title document_type - - - K7 - - - - - - $destination.url$/update - - - - - - - - - - svkul - svkul - - - - - - - svkul - svkul - - - - - - diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei-update.xml similarity index 94% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei-update.xml index 26de9cc673..55be7b1f05 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei-update.xml @@ -1,9 +1,10 @@ + uzei - uzei-update - http://localhost:8080/search/api/admin/v7.0/connected/ - 1 + uzei-full + + $timestamp.url$ diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/inovatika.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/inovatika.xml index 1103c86e72..c2708606ee 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/inovatika.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/inovatika.xml @@ -1,9 +1,11 @@ - + + inovatika + full-crawl-inovatika + inovatika + + http://localhost:8080/search/api/admin/v7.0/connected/ @@ -13,6 +15,7 @@ select pid CURSOR + modified_date 1 @@ -36,7 +39,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/kkp.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/kkp.xml index f892c148f7..f48c5a793e 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/kkp.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/kkp.xml @@ -1,7 +1,10 @@ - + + + kkp + kkp-full + full + @@ -14,6 +17,10 @@ $iteration.user$ $iteration.pass$ + + + modified_date + 1 @@ -66,7 +73,9 @@ - + + PID root_pid dnnt-labels contains-dnnt-labels + kkp diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/knav.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/knav.xml index 045736913c..a21cc0d9eb 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/knav.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/knav.xml @@ -1,17 +1,15 @@ - + knav - full-crawl-knav + knav-full + full $iteration.url$ - select + search PID @@ -21,6 +19,10 @@ $iteration.user$ $iteration.pass$ + + + modified_date + 2 @@ -30,7 +32,7 @@ - select + search 20 $iteration.url$ @@ -47,12 +49,7 @@ $destination.url$ select - - cdk.collection - K7 - - @@ -73,7 +70,9 @@ + PID root_pid dnnt-labels contains-dnnt-labels + knav diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp.xml index 4ecf164382..dd647dffd0 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp.xml @@ -1,12 +1,8 @@ - - + - knav - full-crawl-knav + mlp + mlp-full diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk.xml index 7bc3589279..3bdd988c60 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk.xml @@ -1,7 +1,9 @@ - + + + mzk + mzk-full + @@ -48,22 +50,17 @@ - mzk mzk - + + + pid root.pid licenses contains_licenses licenses_of_ancestors + - mzk diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnnto.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnnto.xml deleted file mode 100644 index e76c46653b..0000000000 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_license_dnnto.xml +++ /dev/null @@ -1,76 +0,0 @@ - - - - - - - licenses:dnnto - - $iteration.url$ - select - pid - 4000 - CURSOR - - - 2 - - - - - $iteration.url$ - select - 20 - - pid - - true - root.pid - pid - - $destination.url$ - select - - - cdk.collection - - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors - - - - - $destination.url$/update - - - - - - - - - - mzk - mzk - mzk_dnnto - dnnto - - - - - - - - mzk - mzk_dnnto - dnnto - - - - - - diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/nfa.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/nfa.xml index a9efbcfa56..9b059f3272 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/nfa.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/nfa.xml @@ -1,11 +1,9 @@ - + nfa - full-crawl-nfa + nfa-full + @@ -21,6 +19,10 @@ $iteration.user$ $iteration.pass$ + + + modified_date + 1 @@ -73,6 +75,7 @@ + PID root_pid dnnt-labels contains-dnnt-labels nfa diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_1.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_1.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_1.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_1.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_2.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_2.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_2.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_2.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_3.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_3.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_3.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_3.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_4.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_4.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_4.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_4.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_5.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_5.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_5.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_5.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_6.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_6.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_6.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_6.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_7.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_7.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_7.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_7.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_8.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_8.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp_sliced_8.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mlp/mlp_sliced_8.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_1.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_1.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_1.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_1.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_10.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_10.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_10.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_10.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_11.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_11.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_11.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_11.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_12.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_12.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_12.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_12.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_2.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_2.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_2.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_2.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_3.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_3.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_3.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_3.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_4.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_4.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_4.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_4.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_5.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_5.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_5.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_5.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_6.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_6.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_6.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_6.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_7.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_7.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_7.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_7.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_8.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_8.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_8.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_8.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_9.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_9.xml similarity index 100% rename from processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mzk_sliceed_9.xml rename to processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/mzk/mzk_sliceed_9.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkhk.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkhk.xml index 208743c5d9..f00b3884b3 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkhk.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkhk.xml @@ -1,12 +1,10 @@ - - + svkhk - full-crawl-svkh + svkh-full + + $timestamp.url$ diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkul.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkul.xml index 56ebf867c9..8d9e985dd9 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkul.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkul.xml @@ -1,7 +1,11 @@ - + + + svkul + svkul-full + + $timestamp.url$ + diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/uzei.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/uzei.xml index 6f8de6c735..55be7b1f05 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/uzei.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/uzei.xml @@ -1,9 +1,11 @@ - + + + uzei + uzei-full + + $timestamp.url$ + diff --git a/search/web/WEB-INF/web.xml b/search/web/WEB-INF/web.xml index 7a9b04d73c..f2d3818e26 100644 --- a/search/web/WEB-INF/web.xml +++ b/search/web/WEB-INF/web.xml @@ -91,7 +91,6 @@ - - From 69d26998471fa37231583ab3a48dbd2d8b94879c Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 9 Jan 2023 12:43:22 +0100 Subject: [PATCH 056/235] Updated descriptors --- .../periodicupdates/configurations/knav-update.xml | 1 - .../periodicupdates/configurations/nfa-update.xml | 1 + .../periodicupdates/configurations/svkul-update.xml | 1 + .../periodicupdates/configurations/uzei-update.xml | 5 ++--- .../services/workers/replicate/configurations/knav.xml | 4 ++-- .../services/workers/replicate/configurations/mlp.xml | 3 --- .../services/workers/replicate/configurations/ntk.xml | 10 +++++----- .../workers/replicate/configurations/svkhk.xml | 9 --------- .../services/workers/replicate/configurations/uzei.xml | 7 ------- 9 files changed, 11 insertions(+), 30 deletions(-) diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav-update.xml index 148e79f51e..07c65de4ec 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav-update.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav-update.xml @@ -18,7 +18,6 @@ modified_date - root_pid:"uuid:477085e4-50f6-4893-a639-229a41fc7a3f" diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa-update.xml index 212df2e4fb..b75f2938f8 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa-update.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa-update.xml @@ -74,6 +74,7 @@ + PID root_pid dnnt-labels contains-dnnt-labels nfa diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul-update.xml index 8d9e985dd9..b223ba3be4 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul-update.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul-update.xml @@ -70,6 +70,7 @@ + PID root_pid dnnt-labels contains-dnnt-labels svkul diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei-update.xml index 55be7b1f05..83eb31ab04 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei-update.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei-update.xml @@ -74,10 +74,9 @@ + PID root_pid dnnt-labels contains-dnnt-labels + - uzei diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/knav.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/knav.xml index a21cc0d9eb..77fa4c1c5b 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/knav.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/knav.xml @@ -9,7 +9,7 @@ $iteration.url$ - search + select PID @@ -32,7 +32,7 @@ - search + select 20 $iteration.url$ diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp.xml index dd647dffd0..b30b96c0e1 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/mlp.xml @@ -69,9 +69,6 @@ - mlp diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/ntk.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/ntk.xml index d65d0801a4..aa9fa9bdce 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/ntk.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/ntk.xml @@ -1,9 +1,9 @@ - + + + mzk + mzk-full + diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkhk.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkhk.xml index f00b3884b3..054fab6e59 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkhk.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/svkhk.xml @@ -4,7 +4,6 @@ svkhk svkh-full - $timestamp.url$ @@ -62,10 +61,6 @@ - svkhk svkhk @@ -77,10 +72,6 @@ svkhk - - diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/uzei.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/uzei.xml index 55be7b1f05..43a7fea2a6 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/uzei.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/uzei.xml @@ -62,10 +62,6 @@ - uzei uzei @@ -75,9 +71,6 @@ - uzei From 3dcf09694c376b9730b156d6e460712e238d56b0 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 31 Jan 2023 23:14:54 +0100 Subject: [PATCH 057/235] Updates --- .../replicate/copy/CopyReplicateWorker.java | 7 ++++++- .../periodicupdates/configurations/mzk-update.xml | 15 +++++++++++++-- .../periodicupdates/configurations/nfa-update.xml | 6 +++++- .../configurations/svkhk-update.xml | 5 ++++- .../configurations/svkul-update.xml | 7 ++++++- .../configurations/uzei-update.xml | 7 ++++++- .../kramerius/timestamps/impl/SolrTimestamp.java | 3 +++ 7 files changed, 43 insertions(+), 7 deletions(-) diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java index 7dcb23fc24..4bcfc24f43 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java @@ -149,7 +149,12 @@ public boolean acceptElement(Element element) { public ModifyFieldResult modifyField(Element field) { List deleteFields = Arrays.asList(); - List addValues = Arrays.asList("licenses","licenses_of_ancestors","contains_licenses"); + List addValues = Arrays.asList( + "licenses", + "licenses_of_ancestors", + "contains_licenses", + "in_collections", + "in_collections.direct"); String name = field.getAttribute("name"); if (deleteFields.contains(name)) { diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/mzk-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/mzk-update.xml index c73c7e455a..24dca68e1e 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/mzk-update.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/mzk-update.xml @@ -3,6 +3,7 @@ mzk mzk-update + update $timestamp.url$ @@ -10,10 +11,17 @@ $iteration.url$ + $iteration.user$ + $iteration.pass$ + select pid 4000 CURSOR + + + indexed + 2 @@ -22,8 +30,11 @@ $iteration.url$ + $iteration.user$ + $iteration.pass$ + select - 20 + 70 pid @@ -59,7 +70,7 @@ - pid root.pid licenses contains_licenses licenses_of_ancestors + pid root.pid licenses contains_licenses licenses_of_ancestors in_collections in_collections.direct mzk diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa-update.xml index b75f2938f8..6bd0e89c4d 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa-update.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/nfa-update.xml @@ -23,6 +23,10 @@ $iteration.user$ $iteration.pass$ + + + modified_date + 1 @@ -33,7 +37,7 @@ select - 20 + 70 $iteration.url$ diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkhk-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkhk-update.xml index a3ae4b90b5..71745626bb 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkhk-update.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkhk-update.xml @@ -3,6 +3,7 @@ svkhk svkh-full + update $timestamp.url$ @@ -19,6 +20,8 @@ $iteration.user$ $iteration.pass$ + + modified_date @@ -30,7 +33,7 @@ select - 20 + 70 $iteration.url$ diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul-update.xml index b223ba3be4..b5dc25c786 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul-update.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/svkul-update.xml @@ -3,6 +3,7 @@ svkul svkul-full + update $timestamp.url$ @@ -18,6 +19,10 @@ $iteration.user$ $iteration.pass$ + + + modified_date + 1 @@ -28,7 +33,7 @@ select - 20 + 70 $iteration.url$ diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei-update.xml index 83eb31ab04..8bdcfb71aa 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei-update.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/uzei-update.xml @@ -3,6 +3,7 @@ uzei uzei-full + update $timestamp.url$ @@ -18,6 +19,10 @@ $iteration.user$ $iteration.pass$ + + + modified_date + 1 @@ -28,7 +33,7 @@ select - 20 + 70 $iteration.url$ diff --git a/shared/common/src/main/java/cz/incad/kramerius/timestamps/impl/SolrTimestamp.java b/shared/common/src/main/java/cz/incad/kramerius/timestamps/impl/SolrTimestamp.java index 0c5e5e6fc1..0107bd0434 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/timestamps/impl/SolrTimestamp.java +++ b/shared/common/src/main/java/cz/incad/kramerius/timestamps/impl/SolrTimestamp.java @@ -1,5 +1,7 @@ package cz.incad.kramerius.timestamps.impl; +import java.text.ParseException; +import java.text.SimpleDateFormat; import java.time.Instant; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; @@ -170,5 +172,6 @@ public static Timestamp fromJSONDoc(String name, JSONObject doc) { return new SolrTimestamp(id, name, type, date, indexed, updated, batches, workers); } + } From c95380a598a721c1f7e540ced0c2be33b2ff1afc Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 1 Feb 2023 00:19:19 +0100 Subject: [PATCH 058/235] Forward url property introduced --- .../redirection/item/V5ForwardHandler.java | 24 +++++++++++++------ .../redirection/item/V7ForwardHandler.java | 24 +++++++++++++------ 2 files changed, 34 insertions(+), 14 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java index b76372fc0e..87537e9e4b 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java @@ -26,11 +26,21 @@ import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.ProxyItemHandler.RequestMethodName; import cz.incad.kramerius.security.User; +import cz.incad.kramerius.utils.conf.KConfiguration; public class V5ForwardHandler extends V5RedirectHandler { public static final Logger LOGGER = Logger.getLogger(V5ForwardHandler.class.getName()); + + + protected String forwardUrl() { + String baseurl = KConfiguration.getInstance().getConfiguration() + .getString("cdk.collections.sources." + this.source + ".forwardurl"); + return baseurl; + } + + public V5ForwardHandler(Instances instances, User user, Client client, SolrAccess solrAccess, String source, String pid, String remoteAddr) { super(instances, user, client, solrAccess, source, pid, remoteAddr); @@ -38,7 +48,7 @@ public V5ForwardHandler(Instances instances, User user, Client client, SolrAcces @Override public Response image(RequestMethodName method) throws ProxyHandlerException { - String baseurl = super.baseUrl(); + String baseurl = this.forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/item/" + this.pid + "/streams/IMG_FULL"; if (method == RequestMethodName.head) { @@ -50,7 +60,7 @@ public Response image(RequestMethodName method) throws ProxyHandlerException { @Override public Response imagePreview(RequestMethodName method) throws ProxyHandlerException { - String baseurl = super.baseUrl(); + String baseurl = this.forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/item/" + this.pid + "/streams/IMG_PREVIEW"; @@ -64,12 +74,12 @@ public Response imagePreview(RequestMethodName method) throws ProxyHandlerExcept @Override public Response mods(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { - String baseurl = super.baseUrl(); + String baseurl = this.baseUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/item/" + this.pid + "/streams/BIBLIO_MODS"; return buildRedirectResponse(url); } else { - String baseurl = super.baseUrl(); + String baseurl = this.forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/item/" + this.pid + "/streams/BIBLIO_MODS"; // String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/item/" @@ -80,7 +90,7 @@ public Response mods(RequestMethodName method) throws ProxyHandlerException { @Override public Response zoomifyImageProperties(RequestMethodName method) throws ProxyHandlerException { - String baseurl = super.baseUrl(); + String baseurl = this.forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/zoomify/" + this.pid + "/ImageProperties.xml"; @@ -93,7 +103,7 @@ public Response zoomifyImageProperties(RequestMethodName method) throws ProxyHan @Override public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHandlerException { - String baseurl = baseUrl(); + String baseurl = this.forwardUrl(); String formatted = String.format("api/v5.0/cdk/forward/zoomify/%s/%s/%s", this.pid, tileGroupStr, tileStr); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + formatted; return buildForwardResponseGET(url); @@ -101,7 +111,7 @@ public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHan @Override public Response textOCR(RequestMethodName method) throws ProxyHandlerException { - String baseurl = super.baseUrl(); + String baseurl = this.forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/" + this.pid + "/streams/TEXT_OCR"; diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index 0c76b5cb63..bc09c1a6a3 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -23,6 +23,7 @@ import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.ProxyItemHandler.RequestMethodName; import cz.incad.kramerius.security.User; +import cz.incad.kramerius.utils.conf.KConfiguration; public class V7ForwardHandler extends V7RedirectHandler { @@ -32,9 +33,16 @@ public V7ForwardHandler(Instances instances, User user, Client client, SolrAcce super(instances, user, client, solrAccess, source, pid, remoteAddr); } + protected String forwardUrl() { + String baseurl = KConfiguration.getInstance().getConfiguration() + .getString("cdk.collections.sources." + this.source + ".forwardurl"); + return baseurl; + } + + @Override public Response image(RequestMethodName method) throws ProxyHandlerException { - String baseurl = super.baseUrl(); + String baseurl = this.forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/item/" + this.pid + "/streams/IMG_FULL"; if (method == RequestMethodName.head) { @@ -46,7 +54,7 @@ public Response image(RequestMethodName method) throws ProxyHandlerException { @Override public Response imagePreview(RequestMethodName method) throws ProxyHandlerException { - String baseurl = super.baseUrl(); + String baseurl = this.forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/item/" + this.pid + "/streams/IMG_PREVIEW"; @@ -62,7 +70,7 @@ public Response mods(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return super.mods(method); } else { - String baseurl = super.baseUrl(); + String baseurl = this.forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/item/" + this.pid + "/streams/BIBLIO_MODS"; return buildForwardResponseGET(url); @@ -81,19 +89,21 @@ public Response zoomifyImageProperties(RequestMethodName method) throws ProxyHan return buildForwardResponseGET(url); } } + @Override public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHandlerException { - String formatted = String.format("image/zoomify/%s/%s", tileGroupStr, tileStr); + //String formatted = String.format("image/zoomify/%s/%s", tileGroupStr, tileStr); //String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/client/v7.0/items/" + pid + "/" + endpoint; - String baseurl = baseUrl(); - String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/client/v7.0/items/" + this.pid + "/"+formatted; + String baseurl = forwardUrl(); + String formatted = String.format("api/cdk/v7.0/forward/zoomify/%s/%s/%s", this.pid, tileGroupStr, tileStr); + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + formatted; return buildRedirectResponse(url); } @Override public Response textOCR(RequestMethodName method) throws ProxyHandlerException { - String baseurl = super.baseUrl(); + String baseurl = this.forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/item/" + this.pid + "/streams/TEXT_OCR"; From 06cbe43e3be575af098a5012e2cf53b8324f770f Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 1 Feb 2023 10:33:01 +0100 Subject: [PATCH 059/235] Fixed forwarded url in user endpoint --- .../v60/redirection/user/V5ForwardUserHandler.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java index 04f0b1b36a..72e8ae0b52 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java @@ -22,6 +22,7 @@ import cz.incad.kramerius.security.User; import cz.incad.kramerius.security.impl.RoleImpl; import cz.incad.kramerius.security.impl.UserImpl; +import cz.incad.kramerius.utils.conf.KConfiguration; public class V5ForwardUserHandler extends ProxyUserHandler { @@ -30,9 +31,16 @@ public V5ForwardUserHandler(Instances instances, User user, Client client, SolrA super(instances, user, client, solrAccess, source, remoteAddr); } + protected String forwardUrl() { + String baseurl = KConfiguration.getInstance().getConfiguration() + .getString("cdk.collections.sources." + this.source + ".forwardurl"); + return baseurl; + } + + @Override public Pair> user() throws ProxyHandlerException { - String baseurl = baseUrl(); + String baseurl = forwardUrl(); //https://kramerius-dnnt.lib.cas.cz/search/api/v5.0/cdk/forward/user String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/user"; ClientResponse fResponse = super.forwardedResponse(url); From 497f2705ef259f26d613a2d1ef1fcf274fb43879 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 1 Feb 2023 12:42:53 +0100 Subject: [PATCH 060/235] Forward headers disabled --- .../apiNew/client/v60/redirection/ProxyHandlerSupport.java | 5 ++++- .../client/v60/redirection/user/V5ForwardUserHandler.java | 4 +++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index f3941d1476..ceaac14c50 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -106,13 +106,16 @@ public void write(OutputStream output) throws IOException, WebApplicationExcepti }; ResponseBuilder respEntity = Response.status(200).entity(stream); + + /* Disable header forward headers.keySet().forEach(key -> { List values = headers.get(key); values.stream().forEach(val -> { respEntity.header(key, val); }); }); - + */ + return respEntity.build(); } else { return Response.status(response.getStatus()).build(); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java index 72e8ae0b52..8b42badd27 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java @@ -45,7 +45,9 @@ public Pair> user() throws ProxyHandlerException { String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/user"; ClientResponse fResponse = super.forwardedResponse(url); String entity = fResponse.getEntity(String.class); - return userFromJSON(new JSONObject(entity)); + JSONObject jObject = new JSONObject(entity); + //LOGGER.info("Collecting user information from = "+url+"; information = "+jObject.toString()); + return userFromJSON(jObject); } public static Pair> userFromJSON(JSONObject json) throws JSONException { From bd7788b6cb8102e5eafafce8d9fa39b9352c1106 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 1 Feb 2023 12:57:33 +0100 Subject: [PATCH 061/235] Fix info endpoint --- .../apiNew/client/v60/redirection/item/V5ForwardHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java index 87537e9e4b..3ee68b6b86 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java @@ -124,7 +124,7 @@ public Response textOCR(RequestMethodName method) throws ProxyHandlerException { @Override public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { - String baseurl = super.baseUrl(); + String baseurl = this.forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/" + this.pid + "/streams/ALTO"; @@ -138,7 +138,7 @@ public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { protected JSONArray providedByLicense() { JSONArray retVal = new JSONArray(); try { - String baseurl = super.baseUrl(); + String baseurl = this.forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/providedBy/" + this.pid; WebResource.Builder r = buidFowrardResponse(url); ClientResponse response = r.get(ClientResponse.class); From c77409376d739c3de8c6fce78977491c2945eabc Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 24 Feb 2023 11:59:32 +0100 Subject: [PATCH 062/235] Default templates --- .../configurations/default_k7-update.xml | 82 +++++++++++++++++ .../configurations/defult_k5-update.xml | 88 +++++++++++++++++++ .../replicate/configurations/default_k5.xml | 86 ++++++++++++++++++ .../replicate/configurations/default_k7.xml | 71 +++++++++++++++ 4 files changed, 327 insertions(+) create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/default_k7-update.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/defult_k5-update.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/default_k7-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/default_k7-update.xml new file mode 100644 index 0000000000..6466d2f198 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/default_k7-update.xml @@ -0,0 +1,82 @@ + + + + $iteration.dl$ + $iteration.dl$-update + update + + $timestamp.url$ + + + + + $iteration.url$ + $iteration.user$ + $iteration.pass$ + + select + pid + 4000 + CURSOR + + + indexed + + + + 2 + + + + + $iteration.url$ + $iteration.user$ + $iteration.pass$ + + select + 70 + + pid + + true + root.pid + pid + + $destination.url$ + select + + + cdk.collection + + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors + + + + + $destination.url$/update + + + + + + + + + + $iteration.dl$ + $iteration.dl$ + + + + + + pid root.pid licenses contains_licenses licenses_of_ancestors in_collections in_collections.direct + + + $iteration.dl$ + + + + + + diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/defult_k5-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/defult_k5-update.xml new file mode 100644 index 0000000000..0f929f9c7b --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/defult_k5-update.xml @@ -0,0 +1,88 @@ + + + + $iteration.dl$ + $iteration.dl$-update + update + + $timestamp.url$ + + + + + $iteration.url$ + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + + 1 + + + + + + + select + 20 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + $iteration.dl$ + $iteration.dl$ + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels + + $iteration.dl$ + + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml new file mode 100644 index 0000000000..eaa137005d --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml @@ -0,0 +1,86 @@ + + + + $iteration.dl$ + $iteration.dl$-full + full + + + + + $iteration.url$ + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + + 1 + + + + + + + select + 80 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + $iteration.dl$ + $iteration.dl$ + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels + + $iteration.dl$ + + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml new file mode 100644 index 0000000000..8e867b944c --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml @@ -0,0 +1,71 @@ + + + + $iteration.dl$ + $iteration.dl$-full + full + + + + + $iteration.url$ + select + pid + 4000 + CURSOR + + + 2 + + + + + $iteration.url$ + select + 80 + + pid + + true + root.pid + pid + + $destination.url$ + select + + + cdk.collection + + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors + + + + + $destination.url$/update + + + + + + + + + + $iteration.dl$ + $iteration.dl$ + + + + + + + pid root.pid licenses contains_licenses licenses_of_ancestors + + + $iteration.dl$ + + + + + + From 515221be7fd9d83be561c0bde58a17423605090f Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 17 Mar 2023 14:16:32 +0100 Subject: [PATCH 063/235] V7 user endpoint support --- .../configurations/knav-update.xml | 10 +- .../workers/replicate/configurations/knav.xml | 1 - .../DefaultOnePropertiesInstance.java | 3 +- .../user/V5ForwardUserHandler.java | 112 +++++++++--------- .../user/V7ForwardUserHandler.java | 104 ++++++++++++++++ .../keycloack/KeycloackUserSupport.java | 16 ++- 6 files changed, 186 insertions(+), 60 deletions(-) create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav-update.xml index 07c65de4ec..6518b2a36c 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav-update.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/knav-update.xml @@ -11,10 +11,13 @@ $iteration.url$ - search + select PID CURSOR + $iteration.user$ + $iteration.pass$ + modified_date @@ -25,9 +28,12 @@ $iteration.url$ - search + select 20 + $iteration.user$ + $iteration.pass$ + PID diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/knav.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/knav.xml index 77fa4c1c5b..42ece51071 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/knav.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/knav.xml @@ -72,7 +72,6 @@ PID root_pid dnnt-labels contains-dnnt-labels - knav diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultOnePropertiesInstance.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultOnePropertiesInstance.java index cefc45086c..4ab6a9de29 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultOnePropertiesInstance.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultOnePropertiesInstance.java @@ -13,6 +13,7 @@ import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.V7RedirectHandler; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.user.ProxyUserHandler; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.user.V5ForwardUserHandler; +import cz.incad.kramerius.rest.apiNew.client.v60.redirection.user.V7ForwardUserHandler; import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.conf.KConfiguration; @@ -82,7 +83,7 @@ public ProxyUserHandler createProxyUserHandler(User user, Client client, SolrAcc case V5: return new V5ForwardUserHandler(this.instances, user, client, solrAccess, source, remoteAddr); default: - throw new UnsupportedOperationException("licence access is not supported"); + return new V7ForwardUserHandler(this.instances, user, client, solrAccess, source, remoteAddr); } } else { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java index 8b42badd27..5194ed3e6d 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java @@ -26,10 +26,10 @@ public class V5ForwardUserHandler extends ProxyUserHandler { - public V5ForwardUserHandler(Instances instances, User user, Client client, SolrAccess solrAccess, String source, - String remoteAddr) { - super(instances, user, client, solrAccess, source, remoteAddr); - } + public V5ForwardUserHandler(Instances instances, User user, Client client, SolrAccess solrAccess, String source, + String remoteAddr) { + super(instances, user, client, solrAccess, source, remoteAddr); + } protected String forwardUrl() { String baseurl = KConfiguration.getInstance().getConfiguration() @@ -37,62 +37,64 @@ protected String forwardUrl() { return baseurl; } - - @Override - public Pair> user() throws ProxyHandlerException { - String baseurl = forwardUrl(); - //https://kramerius-dnnt.lib.cas.cz/search/api/v5.0/cdk/forward/user + @Override + public Pair> user() throws ProxyHandlerException { + String baseurl = forwardUrl(); + // https://kramerius-dnnt.lib.cas.cz/search/api/v5.0/cdk/forward/user String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/user"; ClientResponse fResponse = super.forwardedResponse(url); String entity = fResponse.getEntity(String.class); JSONObject jObject = new JSONObject(entity); - //LOGGER.info("Collecting user information from = "+url+"; information = "+jObject.toString()); + // LOGGER.info("Collecting user information from = "+url+"; information = + // "+jObject.toString()); return userFromJSON(jObject); - } - + } + public static Pair> userFromJSON(JSONObject json) throws JSONException { - if (json.has("id") && json.getInt("id") > -1) { - - Map session = new HashMap<>(); - String loginName = json.optString("lname"); - String firstName = json.optString("firstname"); - String surName = json.optString("surname");; - - List licenses = new ArrayList<>(); - List roles = new ArrayList<>(); - JSONArray rolesJSONArray = json.optJSONArray("roles"); - - if (rolesJSONArray != null) { - for (int i = 0; i < rolesJSONArray.length(); i++) { - String optString = rolesJSONArray.getJSONObject(i).optString("name"); - roles.add(new RoleImpl(optString)); - } - } - - if (json.has("session")) { - JSONObject sessionAttrs = json.getJSONObject("session"); - sessionAttrs.keySet().forEach(key-> { - Object object = sessionAttrs.get((String) key); - session.put(key.toString(), object.toString()); - }); - } - - - if (json.has("labels")) { - JSONArray licensesArray = json.getJSONArray("labels"); - for (int i = 0; i < licensesArray.length(); i++) { - licenses.add(licensesArray.getString(i)); - } - } - - UserImpl userImpl = new UserImpl(1, firstName != null ? firstName : "", surName != null ? surName : "", loginName, 0); - userImpl.setGroups(roles.toArray(new Role[roles.size()])); - session.entrySet().forEach(entry-> { - userImpl.addSessionAttribute(entry.getKey(), entry.getValue()); - }); - - return Pair.of(userImpl, licenses); - - } else return null; + if (json.has("id") && json.getInt("id") > -1) { + + Map session = new HashMap<>(); + String loginName = json.optString("lname"); + String firstName = json.optString("firstname"); + String surName = json.optString("surname"); + ; + + List licenses = new ArrayList<>(); + List roles = new ArrayList<>(); + JSONArray rolesJSONArray = json.optJSONArray("roles"); + + if (rolesJSONArray != null) { + for (int i = 0; i < rolesJSONArray.length(); i++) { + String optString = rolesJSONArray.getJSONObject(i).optString("name"); + roles.add(new RoleImpl(optString)); + } + } + + if (json.has("session")) { + JSONObject sessionAttrs = json.getJSONObject("session"); + sessionAttrs.keySet().forEach(key -> { + Object object = sessionAttrs.get((String) key); + session.put(key.toString(), object.toString()); + }); + } + + if (json.has("labels")) { + JSONArray licensesArray = json.getJSONArray("labels"); + for (int i = 0; i < licensesArray.length(); i++) { + licenses.add(licensesArray.getString(i)); + } + } + + UserImpl userImpl = new UserImpl(1, firstName != null ? firstName : "", surName != null ? surName : "", + loginName, 0); + userImpl.setGroups(roles.toArray(new Role[roles.size()])); + session.entrySet().forEach(entry -> { + userImpl.addSessionAttribute(entry.getKey(), entry.getValue()); + }); + + return Pair.of(userImpl, licenses); + + } else + return null; } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java new file mode 100644 index 0000000000..359c343b86 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java @@ -0,0 +1,104 @@ +package cz.incad.kramerius.rest.apiNew.client.v60.redirection.user; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang3.tuple.Pair; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; + +import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; +import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; +import cz.incad.kramerius.security.Role; +import cz.incad.kramerius.security.User; +import cz.incad.kramerius.security.impl.RoleImpl; +import cz.incad.kramerius.security.impl.UserImpl; +import cz.incad.kramerius.utils.conf.KConfiguration; + +public class V7ForwardUserHandler extends ProxyUserHandler { + + private static final String NAME_KEY = "name"; + private static final String SESSION_KEY = "session"; + private static final String LICENSES_KEY = "licenses"; + private static final String ROLES_KEY = "roles"; + private static final String AUTHENTICATED_KEY = "authenticated"; + private static final String UID_KEY = "uid"; + + public V7ForwardUserHandler(Instances instances, User user, Client client, SolrAccess solrAccess, String source, + String remoteAddr) { + super(instances, user, client, solrAccess, source, remoteAddr); + } + + protected String forwardUrl() { + String baseurl = KConfiguration.getInstance().getConfiguration() + .getString("cdk.collections.sources." + this.source + ".forwardurl"); + return baseurl; + } + + @Override + public Pair> user() throws ProxyHandlerException { + String baseurl = forwardUrl(); + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/user"; + ClientResponse fResponse = super.forwardedResponse(url); + String entity = fResponse.getEntity(String.class); + JSONObject jObject = new JSONObject(entity); + return userFromJSON(jObject); + } + + public static Pair> userFromJSON(JSONObject json) throws JSONException { + if (json.optBoolean(AUTHENTICATED_KEY, false)) { + String uid = json.getString(UID_KEY); + String name = json.optString(NAME_KEY,""); + String[] names = name.split("\\s"); + String firstName = null; + String secondName = null; + if (names.length > 0) firstName = names[0]; + if (names.length > 1) secondName = names[1]; + + Map session = new HashMap<>(); + List licenses = new ArrayList<>(); + List roles = new ArrayList<>(); + JSONArray rolesJSONArray = json.optJSONArray(ROLES_KEY); + + if (rolesJSONArray != null) { + for (int i = 0; i < rolesJSONArray.length(); i++) { + String optString = rolesJSONArray.getString(i); + roles.add(new RoleImpl(optString)); + } + } + + if (json.has(SESSION_KEY)) { + JSONObject sessionAttrs = json.getJSONObject(SESSION_KEY); + sessionAttrs.keySet().forEach(key -> { + Object object = sessionAttrs.get((String) key); + session.put(key.toString(), object.toString()); + }); + } + + if (json.has(LICENSES_KEY)) { + JSONArray licensesArray = json.getJSONArray(LICENSES_KEY); + for (int i = 0; i < licensesArray.length(); i++) { + licenses.add(licensesArray.getString(i)); + } + } + + UserImpl userImpl = new UserImpl(1, firstName != null ? firstName : "", secondName != null ? secondName : "", uid, 0); + userImpl.setGroups(roles.toArray(new Role[roles.size()])); + + session.entrySet().forEach(entry -> { + userImpl.addSessionAttribute(entry.getKey(), entry.getValue()); + }); + + return Pair.of(userImpl, licenses); + + } else + return null; + } +} diff --git a/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java b/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java index 2cf895732f..27afe3b652 100644 --- a/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java +++ b/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java @@ -166,6 +166,20 @@ protected Keycloack3rdUser createUserWrapper(HttpServletRequest req, String user @Override public String calculateUserName(HttpServletRequest request) { - return KEYCLOACK_USER_PREFIX+ request.getUserPrincipal().getName(); + if (request.getUserPrincipal() != null) { + if (request.getUserPrincipal() instanceof KeycloakPrincipal) { + AccessToken token = ((KeycloakPrincipal) request.getUserPrincipal()).getKeycloakSecurityContext() + .getToken(); + if (token.getEmail() != null) { + return token.getEmail(); + } else { + return token.getPreferredUsername(); + } + } else { + return request.getUserPrincipal().getName(); + } + } else { + return "null"; + } } } From 15eafdc57ad48213fabf591e008b11424650aab7 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 17 Mar 2023 15:12:19 +0100 Subject: [PATCH 064/235] Forward in zoomify --- .../apiNew/client/v60/redirection/item/V7ForwardHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index bc09c1a6a3..e5da376895 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -97,7 +97,7 @@ public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHan String baseurl = forwardUrl(); String formatted = String.format("api/cdk/v7.0/forward/zoomify/%s/%s/%s", this.pid, tileGroupStr, tileStr); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + formatted; - return buildRedirectResponse(url); + return buildForwardResponseGET(url); } From c5973da80acd7f2f01b7c474264d4523b3c8ec07 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 27 Apr 2023 22:15:08 +0200 Subject: [PATCH 065/235] Forward or not forward shibboleth attributes --- .../replicate/configurations/default_k7.xml | 7 +++++-- .../v60/redirection/ProxyHandlerSupport.java | 18 ++++++++++++++---- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml index 8e867b944c..6ade81cdee 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml @@ -13,6 +13,9 @@ pid 4000 CURSOR + + $iteration.fquery$ + 2 @@ -36,7 +39,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* titles.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors @@ -59,7 +62,7 @@ - pid root.pid licenses contains_licenses licenses_of_ancestors + pid root.pid licenses contains_licenses licenses_of_ancestors titles.* collection.* in_collections in_collections.* title.* titles.* text_ocr $iteration.dl$ diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index ceaac14c50..4eb46f9fe4 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -158,10 +158,20 @@ protected void mockSession() { protected WebResource.Builder buidFowrardResponse(String url) { String prefixHeaders = KConfiguration.getInstance().getConfiguration().getString("cdk.shibboleth.forward.headers"); - Map attributes = this.user.getSessionAttributes(); - String header = attributes.keySet().stream().map(key -> { - return "header_" + key + "=" + attributes.get(key); - }).collect(Collectors.joining("|")); + // no user session attributes in case of no federation + String header = ""; + + boolean shibbolethAttributes = KConfiguration.getInstance().getConfiguration() + .getBoolean("cdk.collections.sources." + this.source + ".shibboleth_attributes", false); + + //cdk.collections.sources.svkul.pswd=yTTJ00B + + if (shibbolethAttributes) { + Map attributes = this.user.getSessionAttributes(); + header = header + attributes.keySet().stream().map(key -> { + return "header_" + key + "=" + attributes.get(key); + }).collect(Collectors.joining("|")); + } if (this.remoteAddr != null) { header = header + "|" + "header_ip_address=" + this.remoteAddr; From 62672879c28631a0225e4c3c30bb58492d4c05b4 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 28 Apr 2023 06:09:18 +0200 Subject: [PATCH 066/235] Default value for forward shib attributes --- .../apiNew/client/v60/redirection/ProxyHandlerSupport.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 4eb46f9fe4..fc9a9ed55c 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -162,9 +162,8 @@ protected WebResource.Builder buidFowrardResponse(String url) { String header = ""; boolean shibbolethAttributes = KConfiguration.getInstance().getConfiguration() - .getBoolean("cdk.collections.sources." + this.source + ".shibboleth_attributes", false); + .getBoolean("cdk.collections.sources." + this.source + ".shibboleth_attributes", true); - //cdk.collections.sources.svkul.pswd=yTTJ00B if (shibbolethAttributes) { Map attributes = this.user.getSessionAttributes(); From 4b66bd86395f9e2ef3396d3fe0f6082e2013f6d4 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 5 May 2023 15:50:18 +0200 Subject: [PATCH 067/235] V7 forward handler --- .../apiNew/client/v60/redirection/item/V7ForwardHandler.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index e5da376895..f9dd47e873 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -84,8 +84,8 @@ public Response zoomifyImageProperties(RequestMethodName method) throws ProxyHan } else { String baseurl = super.baseUrl(); - String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/client/v7.0/items/" + this.pid - + "/image/zoomify/ImageProperties.xml"; + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/zoomify/" + this.pid + + "/ImageProperties.xml"; return buildForwardResponseGET(url); } } From 88095ad8d7f47b6e661ae08912756e575c063fe9 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 5 May 2023 16:09:59 +0200 Subject: [PATCH 068/235] Forward resource --- .../apiNew/client/v60/redirection/item/V7ForwardHandler.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index f9dd47e873..4ed44f4c05 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -83,9 +83,10 @@ public Response zoomifyImageProperties(RequestMethodName method) throws ProxyHan return super.zoomifyImageProperties(method); } else { - String baseurl = super.baseUrl(); + String baseurl = forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/zoomify/" + this.pid + "/ImageProperties.xml"; + LOGGER.info(String.format("Base url %s", url)); return buildForwardResponseGET(url); } } From 4b615a4bf37569722d7d7d250e07b9a58e8fa250 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 5 May 2023 16:45:35 +0200 Subject: [PATCH 069/235] info resource --- .../redirection/item/V7ForwardHandler.java | 48 ++++++++++++++++++- 1 file changed, 47 insertions(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index 4ed44f4c05..6eab9f48d7 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -2,10 +2,15 @@ import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.util.logging.Level; import java.util.logging.Logger; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; +import javax.ws.rs.core.StreamingOutput; +import javax.ws.rs.core.Response.ResponseBuilder; import org.apache.commons.io.IOUtils; import org.json.JSONArray; @@ -40,6 +45,48 @@ protected String forwardUrl() { } + + + @Override + public Response info() throws ProxyHandlerException { + + + //http://tunel/search/api/cdk/v7.0/forward/providedBy/uuid:dfc71c54-0fff-4e8a-a59e-b235274da271 + JSONArray licenses = null; + String baseurl = this.forwardUrl(); + String providedByUrl = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/item/providedBy/" + this.pid; + WebResource.Builder providedByBuilder = buidFowrardResponse(providedByUrl); + ClientResponse providedBy = providedByBuilder.get(ClientResponse.class); + if (providedBy.getStatus() == 200) { + String content = providedBy.getEntity(String.class); + //{"licenses":["dnnto"]} + JSONObject providedByJSON = new JSONObject(content); + licenses = providedByJSON.optJSONArray("licenses"); + } + + + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/client/v7.0/items/" + this.pid + "/info"; + // enhance by providedBy + WebResource.Builder b = buidFowrardResponse(url); + ClientResponse response = b.get(ClientResponse.class); + if (response.getStatus() == 200) { + + //"providedByLicenses": ["dnnto"], + String infoContent = response.getEntity(String.class); + JSONObject infoContentJSON = new JSONObject(infoContent); + if (licenses != null) infoContentJSON.put("providedByLicenses", licenses); + + + ResponseBuilder respEntity = Response.status(200).entity(infoContentJSON.toString()); + + return respEntity.build(); + } else { + return Response.status(response.getStatus()).build(); + } + + //return super.info(); + } + @Override public Response image(RequestMethodName method) throws ProxyHandlerException { String baseurl = this.forwardUrl(); @@ -86,7 +133,6 @@ public Response zoomifyImageProperties(RequestMethodName method) throws ProxyHan String baseurl = forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/zoomify/" + this.pid + "/ImageProperties.xml"; - LOGGER.info(String.format("Base url %s", url)); return buildForwardResponseGET(url); } } From b0bc57d03217e87f4adae3a8f76bd71cc0b39094 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 5 May 2023 17:53:33 +0200 Subject: [PATCH 070/235] Debug messages --- .../client/v60/redirection/item/V7ForwardHandler.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index 6eab9f48d7..3fe2e6c5cd 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -55,6 +55,8 @@ public Response info() throws ProxyHandlerException { JSONArray licenses = null; String baseurl = this.forwardUrl(); String providedByUrl = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/item/providedBy/" + this.pid; + LOGGER.info("Provided by url "+providedByUrl); + WebResource.Builder providedByBuilder = buidFowrardResponse(providedByUrl); ClientResponse providedBy = providedByBuilder.get(ClientResponse.class); if (providedBy.getStatus() == 200) { @@ -63,8 +65,8 @@ public Response info() throws ProxyHandlerException { JSONObject providedByJSON = new JSONObject(content); licenses = providedByJSON.optJSONArray("licenses"); } - - + //TOD + LOGGER.info("Provided by result "+licenses); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/client/v7.0/items/" + this.pid + "/info"; // enhance by providedBy WebResource.Builder b = buidFowrardResponse(url); From 6def63a62b7c034e99986eead2030d9eab30cd38 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 5 May 2023 18:25:21 +0200 Subject: [PATCH 071/235] Fix url --- .../apiNew/client/v60/redirection/item/V7ForwardHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index 3fe2e6c5cd..f51cf2dd1e 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -54,7 +54,7 @@ public Response info() throws ProxyHandlerException { //http://tunel/search/api/cdk/v7.0/forward/providedBy/uuid:dfc71c54-0fff-4e8a-a59e-b235274da271 JSONArray licenses = null; String baseurl = this.forwardUrl(); - String providedByUrl = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/item/providedBy/" + this.pid; + String providedByUrl = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/providedBy/" + this.pid; LOGGER.info("Provided by url "+providedByUrl); WebResource.Builder providedByBuilder = buidFowrardResponse(providedByUrl); From 23aac59bf2e59c5072a40180eec2754b125d27cd Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 19 May 2023 08:29:07 +0200 Subject: [PATCH 072/235] Titles search in updates --- .../periodicupdates/configurations/default_k7-update.xml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/default_k7-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/default_k7-update.xml index 6466d2f198..46e808ec67 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/default_k7-update.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/default_k7-update.xml @@ -47,8 +47,10 @@ cdk.collection - + + indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* titles.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors From 1a3db87d82b8cebc193b2fb6e2394006dc98a890 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 25 May 2023 08:05:40 +0200 Subject: [PATCH 073/235] auth endpoints --- .../configurations/mzk_license.xml | 51 +++++++++ .../rest/api/guice/ApiServletModule.java | 3 +- .../client/v60/ClientKeycloakConfig.java | 88 ++++++++++++++ .../apiNew/client/v60/ClientUserResource.java | 85 +++++++++++++- .../rest/apiNew/client/v70/InfoResource.java | 107 ++++++++++++++++++ 5 files changed, 331 insertions(+), 3 deletions(-) create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/updatefield/configurations/mzk_license.xml create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientKeycloakConfig.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v70/InfoResource.java diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/updatefield/configurations/mzk_license.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/updatefield/configurations/mzk_license.xml new file mode 100644 index 0000000000..8ac4a8eeef --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/updatefield/configurations/mzk_license.xml @@ -0,0 +1,51 @@ + + + + + + + licenses:* + $iteration.url$ + select + pid + 8000 + CURSOR + + + 1 + + + + + $iteration.url$ + select + $iteration.batchsize$ + + pid + + true + root.pid + pid + + $destination.url$ + select + + cdk.collection + + $iteration.fieldlist$ + + + + + + $destination.url$/update + + $destination.updatefield$ + + $destination.update$ + + + + diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java index fdd4a3f699..dc5690eb22 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java @@ -109,6 +109,7 @@ protected void configureServlets() { bind(cz.incad.kramerius.rest.apiNew.client.v60.InfoResource.class); bind(cz.incad.kramerius.rest.apiNew.client.v60.SearchResource.class); bind(cz.incad.kramerius.rest.apiNew.client.v60.ConfigResource.class); + bind(cz.incad.kramerius.rest.apiNew.client.v70.InfoResource.class); bind(Client.class).annotatedWith(Names.named("forward-client")).toProvider(ClientProvider.class).asEagerSingleton(); bind(cz.incad.kramerius.rest.apiNew.client.v60.ItemsResource.class); @@ -135,7 +136,7 @@ protected void configureServlets() { bind(SimpleJSONMessageBodyWriter.class); // decorators - decorators(); + //decorators(); // cdk diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientKeycloakConfig.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientKeycloakConfig.java new file mode 100644 index 0000000000..b3c7410446 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientKeycloakConfig.java @@ -0,0 +1,88 @@ +package cz.incad.kramerius.rest.apiNew.client.v70; + +import org.json.JSONObject; + +import cz.incad.kramerius.utils.StringUtils; + +public class ClientKeycloakConfig { + + private String realm; + private String authServer; + private String resource; + private String secret; + + + public ClientKeycloakConfig(String realm, String authServer, String resource, String secret) { + super(); + this.realm = realm; + this.authServer = authServer; + this.resource = resource; + this.secret = secret; + } + + public String getRealm() { + return realm; + } + + public String getAuthServer() { + return authServer; + } + + public String getResource() { + return resource; + } + + public String getSecret() { + return secret; + } + + public String loginKeycloak(String redirectUrl) { + StringBuilder builder = new StringBuilder(); + builder.append(this.authServer); + if (!builder.toString().endsWith("/")) builder.append("/"); + builder.append("realms/"); + builder.append(this.realm); + builder.append("/protocol/openid-connect/auth?client_id=").append(this.resource); + if (StringUtils.isAnyString(redirectUrl)) { + builder.append("&redirect_uri="+redirectUrl); + } + builder.append("&response_type=code"); + return builder.toString(); + } + + public String logoutKeycloak(String redirectUrl) { + StringBuilder builder = new StringBuilder(); + builder.append(this.authServer); + if (!builder.toString().endsWith("/")) builder.append("/"); + builder.append("realms/"); + builder.append(this.realm); + builder.append("/protocol/openid-connect/logout"); + if (StringUtils.isAnyString(redirectUrl)) { + builder.append("?redirect_uri="+redirectUrl); + } + return builder.toString(); + } + + public String token(String code) { + StringBuilder builder = new StringBuilder(); + builder.append(this.authServer); + if (!builder.toString().endsWith("/")) builder.append("/"); + builder.append("realms/"); + builder.append(this.realm); + builder.append("/protocol/openid-connect/token"); + return builder.toString(); + } + + public static ClientKeycloakConfig load(JSONObject conf) { + String realm = conf.optString("realm"); + String authServer = conf.optString("auth-server-url"); + String resource = conf.optString("resource"); + String secret = null; + JSONObject credentials = conf.optJSONObject("credentials"); + if (credentials != null) { + secret = credentials.optString("secret"); + } + return new ClientKeycloakConfig(realm, authServer, resource, secret); + } + +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientUserResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientUserResource.java index 34b48815ec..349debdfbc 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientUserResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientUserResource.java @@ -16,8 +16,10 @@ */ package cz.incad.kramerius.rest.apiNew.client.v60; +import java.io.FileInputStream; import java.io.IOException; import java.io.UnsupportedEncodingException; +import java.net.URI; import java.security.NoSuchAlgorithmException; import java.sql.SQLException; import java.util.*; @@ -42,6 +44,8 @@ import cz.incad.kramerius.utils.StringUtils; import cz.incad.kramerius.utils.conf.KConfiguration; +import org.apache.commons.io.IOUtils; + import org.apache.commons.lang3.tuple.Pair; import org.json.JSONArray; import org.json.JSONException; @@ -50,6 +54,8 @@ import com.google.inject.Inject; import com.google.inject.Provider; import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.rest.api.exceptions.BadRequestException; import cz.incad.kramerius.rest.api.exceptions.GenericApplicationException; @@ -66,6 +72,12 @@ import cz.incad.kramerius.users.UserProfile; import cz.incad.kramerius.users.UserProfileManager; +import org.jboss.resteasy.specimpl.MultivaluedMapImpl; + + +import static cz.incad.kramerius.Constants.WORKING_DIR; + + //@Path("/v5.0/user") @Path("/client/v7.0/user") public class ClientUserResource { @@ -74,9 +86,13 @@ public class ClientUserResource { public static final Logger LOGGER = Logger.getLogger(ClientUserResource.class.getName()); - public static final String[] LABELS_CRITERIA = new String[]{ + public static final String[] LICENSES_CRITERIA = new String[]{ "cz.incad.kramerius.security.impl.criteria.ReadDNNTLabels", - "cz.incad.kramerius.security.impl.criteria.ReadDNNTLabelsIPFiltered" + "cz.incad.kramerius.security.impl.criteria.ReadDNNTLabelsIPFiltered", + "cz.incad.kramerius.security.impl.criteria.Licenses", + "cz.incad.kramerius.security.impl.criteria.LicensesIPFiltered", + "cz.incad.kramerius.security.impl.criteria.LicensesGEOIPFiltered" + }; @Inject @@ -343,6 +359,69 @@ public Response changePassword(JSONObject rawdata) { + @GET + @Path("auth/login") + public Response login(@QueryParam("redirect_uri") String redirectUri) { + try { + String path = WORKING_DIR + "/keycloak.json"; + String str = IOUtils.toString(new FileInputStream(path),"UTF-8"); + ClientKeycloakConfig cnf = ClientKeycloakConfig.load(new JSONObject(str)); + URI uri = URI.create(cnf.loginKeycloak(redirectUri)); + return Response.temporaryRedirect(uri).build(); + } catch (IOException e) { + throw new GenericApplicationException(e.getMessage()); + } + } + + @GET + @Path("auth/token") + @Produces({ MediaType.APPLICATION_JSON + ";charset=utf-8" }) + public Response token(@QueryParam("code") String code, @QueryParam("redirect_uri") String redirectUri) { + try { + String path = WORKING_DIR + "/keycloak.json"; + String str = IOUtils.toString(new FileInputStream(path),"UTF-8"); + ClientKeycloakConfig cnf = ClientKeycloakConfig.load(new JSONObject(str)); + + String type = "application/x-www-form-urlencoded; charset=UTF-8"; + + Client client = Client.create(); + WebResource webResource = client.resource(cnf.token(code)); + MultivaluedMapImpl values = new MultivaluedMapImpl(); + values.add("grant_type", "authorization_code"); + values.add("code", code); + values.add("client_id", cnf.getResource()); + values.add("client_secret", cnf.getSecret()); + values.add("redirect_uri", redirectUri); + // TODO: Dat to do konfigurace + //values.add("scope", "openid"); + ClientResponse post = webResource.type(type).post(ClientResponse.class, values); + String entity = (String) post.getEntity(String.class); + return Response.ok().entity(entity.toString()).build(); + + } catch (Exception e) { + throw new GenericApplicationException(e.getMessage()); + } + } + + + + + @GET + @Path("auth/logout") + public Response logout(@QueryParam("redirect_uri") String redirectUri) { + try { + String path = WORKING_DIR + "/keycloak.json"; + String str = IOUtils.toString(new FileInputStream(path),"UTF-8"); + ClientKeycloakConfig cnf = ClientKeycloakConfig.load(new JSONObject(str)); + URI uri = URI.create(cnf.logoutKeycloak(redirectUri)); + return Response.temporaryRedirect(uri).build(); + } catch (IOException e) { + throw new GenericApplicationException(e.getMessage()); + } + } + + // Legacy logout - support for old client. + // Will be removed in future @GET @Path("logout") public Response logout() { @@ -350,5 +429,7 @@ public Response logout() { httpServletRequest.getSession().invalidate(); return Response.ok().entity("{}").build(); } + + } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v70/InfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v70/InfoResource.java new file mode 100644 index 0000000000..55375639b2 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v70/InfoResource.java @@ -0,0 +1,107 @@ +package cz.incad.kramerius.rest.apiNew.client.v70; + +import com.google.inject.Inject; +import cz.incad.kramerius.rest.apiNew.exceptions.InternalErrorException; +import cz.incad.kramerius.service.ResourceBundleService; +import cz.incad.kramerius.service.TextsService; +import cz.incad.kramerius.utils.conf.KConfiguration; +import cz.kramerius.searchIndex.indexer.execution.Indexer; +import org.json.JSONObject; + +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import java.io.IOException; +import java.io.InputStream; +import java.util.HashMap; +import java.util.Locale; +import java.util.Properties; +import java.util.logging.Level; +import java.util.logging.Logger; + +/** + * @see cz.incad.kramerius.rest.api.k5.client.info.InfoResource + */ +@Path("/client/v7.0/info") +public class InfoResource extends ClientApiResource { + + public static Logger LOGGER = Logger.getLogger(InfoResource.class.getName()); + + private static HashMap LOCALES = new HashMap() {{ + put("en", Locale.ENGLISH); + put("cs", new Locale("cs", "cz")); + }}; + + @Inject + private TextsService textService; + + @Inject + private ResourceBundleService resourceBundleService; + + @GET + @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8") + public Response getItems(@QueryParam("language") String langCode) { + try { + JSONObject json = new JSONObject(); + json.put("pdfMaxRange", getPdfMaxRange()); + json.put("version", getVersion()); + json.put("hash", getHash()); + //TODO: tohle asi vyhodit, uz resime v ConfigResource + if (langCode != null && LOCALES.containsKey(langCode)) { + json.put("rightMsg", getRightMsg(LOCALES.get(langCode))); + } else { + JSONObject rightMsg = new JSONObject(); + for (String code : LOCALES.keySet()) { + rightMsg.put(code, getRightMsg(LOCALES.get(code))); + } + json.put("rightMsg", rightMsg); + } + //json.put("indexerVersion", Indexer.INDEXER_VERSION); + return Response.ok(json).build(); + } catch (WebApplicationException e) { + throw e; + } catch (Throwable e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new InternalErrorException(e.getMessage()); + } + } + + private String getPdfMaxRange() { + String maxRange = KConfiguration.getInstance().getProperty("generatePdfMaxRange"); + boolean disabled = KConfiguration.getInstance().getConfiguration().getBoolean("turnOffPdfCheck"); + return disabled ? "unlimited" : maxRange; + } + + private String getVersion() throws IOException { + Properties buildProperties = new Properties(); + InputStream revisions = this.getClass().getClassLoader().getResourceAsStream("build.properties"); + if (revisions != null) { + buildProperties.load(revisions); + return buildProperties.getProperty("version"); + } else { + LOGGER.warning("build.properties is not present"); + return ""; + } + } + + private String getHash() throws IOException { + Properties buildProperties = new Properties(); + InputStream revisions = this.getClass().getClassLoader().getResourceAsStream("build.properties"); + if (revisions != null) { + buildProperties.load(revisions); + return buildProperties.getProperty("hash"); + } else { + LOGGER.warning("build.properties is not present"); + return ""; + } + } + @Deprecated //TODO: replace with database (table CONFIG with columns KEY and VALUE) + private String getRightMsg(Locale locale) throws IOException { + String key = "rightMsg"; + if (textService.isAvailable(key, locale)) { + return textService.getText(key, locale); + } else { + return resourceBundleService.getResourceBundle("labels", locale).getString(key); + } + } +} From 5c6e7f12673a69e03fdd4b1699fefb505486db7e Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 25 May 2023 08:11:32 +0200 Subject: [PATCH 074/235] Info resource --- .../cz/incad/kramerius/rest/apiNew/client/v70/InfoResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v70/InfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v70/InfoResource.java index 55375639b2..d741b29c2c 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v70/InfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v70/InfoResource.java @@ -23,7 +23,7 @@ * @see cz.incad.kramerius.rest.api.k5.client.info.InfoResource */ @Path("/client/v7.0/info") -public class InfoResource extends ClientApiResource { +public class InfoResource { public static Logger LOGGER = Logger.getLogger(InfoResource.class.getName()); From 6858295259e3b82dfe8fd9a4473a401588fbc21e Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 25 May 2023 08:44:28 +0200 Subject: [PATCH 075/235] Info resources --- .../kramerius/rest/apiNew/client/v60/ClientUserResource.java | 1 + .../rest/apiNew/client/{v60 => v70}/ClientKeycloakConfig.java | 0 .../cz/incad/kramerius/rest/apiNew/client/v70/InfoResource.java | 1 - 3 files changed, 1 insertion(+), 1 deletion(-) rename rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/{v60 => v70}/ClientKeycloakConfig.java (100%) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientUserResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientUserResource.java index 349debdfbc..4e7316776f 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientUserResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientUserResource.java @@ -65,6 +65,7 @@ import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.user.ProxyUserHandler; +import cz.incad.kramerius.rest.apiNew.client.v70.ClientKeycloakConfig; import cz.incad.kramerius.security.RightsResolver; import cz.incad.kramerius.security.User; import cz.incad.kramerius.security.UserManager; diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientKeycloakConfig.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v70/ClientKeycloakConfig.java similarity index 100% rename from rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientKeycloakConfig.java rename to rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v70/ClientKeycloakConfig.java diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v70/InfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v70/InfoResource.java index d741b29c2c..83d27217aa 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v70/InfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v70/InfoResource.java @@ -5,7 +5,6 @@ import cz.incad.kramerius.service.ResourceBundleService; import cz.incad.kramerius.service.TextsService; import cz.incad.kramerius.utils.conf.KConfiguration; -import cz.kramerius.searchIndex.indexer.execution.Indexer; import org.json.JSONObject; import javax.ws.rs.*; From c418cc4ebffa9925a9b62c4098da36d127b3dd48 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 25 May 2023 09:30:03 +0200 Subject: [PATCH 076/235] decorators / backward compatibility --- .../cz/incad/kramerius/rest/api/guice/ApiServletModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java index dc5690eb22..e1cba59bc6 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java @@ -136,7 +136,7 @@ protected void configureServlets() { bind(SimpleJSONMessageBodyWriter.class); // decorators - //decorators(); + decorators(); // cdk From 9d4242bc89e064e0880bd0a7eb5cd509842b6b28 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 25 May 2023 10:05:33 +0200 Subject: [PATCH 077/235] Disable move info endpoint --- .../cz/incad/kramerius/rest/api/guice/ApiServletModule.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java index e1cba59bc6..cb7133f52c 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java @@ -109,7 +109,7 @@ protected void configureServlets() { bind(cz.incad.kramerius.rest.apiNew.client.v60.InfoResource.class); bind(cz.incad.kramerius.rest.apiNew.client.v60.SearchResource.class); bind(cz.incad.kramerius.rest.apiNew.client.v60.ConfigResource.class); - bind(cz.incad.kramerius.rest.apiNew.client.v70.InfoResource.class); + //bind(cz.incad.kramerius.rest.apiNew.client.v70.InfoResource.class); bind(Client.class).annotatedWith(Names.named("forward-client")).toProvider(ClientProvider.class).asEagerSingleton(); bind(cz.incad.kramerius.rest.apiNew.client.v60.ItemsResource.class); From 9f1d5f12acedf732a699fa1c914c19ca2f7ddff2 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 25 May 2023 10:29:10 +0200 Subject: [PATCH 078/235] Build gradle properties --- search/build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search/build.gradle b/search/build.gradle index ac980ad28a..410a81727a 100644 --- a/search/build.gradle +++ b/search/build.gradle @@ -101,7 +101,7 @@ task infofile(dependsOn: gitcall) { } } -processResources.dependsOn infofile +//processResources.dependsOn infofile war { baseName='search' From 071a23459ef31b428ef860508f34ddb8f05a6ba7 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 25 May 2023 10:30:00 +0200 Subject: [PATCH 079/235] move version & package to gradle.properties --- build.gradle | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index ae8667077f..a3bc5f0235 100644 --- a/build.gradle +++ b/build.gradle @@ -64,10 +64,11 @@ repositories { mavenLocal() } +/* allprojects { group='cz.incad.kramerius' version='7.0.3-beta2_cdk' -} +}*/ /** From 714b36abcd965a132e34c2c144be660838f8b817 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 25 May 2023 10:34:56 +0200 Subject: [PATCH 080/235] Changed version - dev3_cdk --- gradle.properties | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 gradle.properties diff --git a/gradle.properties b/gradle.properties new file mode 100644 index 0000000000..05c0422e33 --- /dev/null +++ b/gradle.properties @@ -0,0 +1,4 @@ +#org.gradle.java.home=c:\\Users\\happy\\Programs\\jdk18 + +group=cz.incad.kramerius +version=7.0.23_dev3_cdk From c47cf96ac478608699adbabb8b9b461042779712 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 18 Sep 2023 15:49:42 +0200 Subject: [PATCH 081/235] IIIF support for K5 --- .../rest/apiNew/client/v60/ItemsResource.java | 89 +++++++++++++++ .../redirection/item/ProxyItemHandler.java | 5 + .../redirection/item/V5ForwardHandler.java | 32 ++++++ .../redirection/item/V5RedirectHandler.java | 20 ++++ .../redirection/item/V7RedirectHandler.java | 20 +++- .../incad/Kramerius/imaging/IiifServlet.java | 102 +++++++++--------- 6 files changed, 217 insertions(+), 51 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java index 7c4791d1c4..3f4e5f2ff0 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java @@ -30,6 +30,7 @@ import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.ProxyItemHandler; +import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.ProxyItemHandler.RequestMethodName; import cz.incad.kramerius.rest.apiNew.exceptions.BadRequestException; import cz.incad.kramerius.rest.apiNew.exceptions.InternalErrorException; import cz.incad.kramerius.security.RightsResolver; @@ -1208,4 +1209,92 @@ public Response getAudioWav(@PathParam("pid") String pid,@PathParam("source") St } } + + @GET + @Path("iiif/{pid}/info.json") + @Produces("application/ld+json") + public Response iiiFManifest(@PathParam("pid") String pid) { + try { + ProxyItemHandler redirectHandler = findRedirectHandler(pid,null); + if (redirectHandler != null) { + return redirectHandler.iiifInfo(RequestMethodName.get, pid); + } else { + return Response.ok().build(); + } + } catch (WebApplicationException e) { + throw e; + } catch (Throwable e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new InternalErrorException(e.getMessage()); + } + } + + @GET + @Path("iiif/{source}/{pid}/info.json") + @Produces("application/ld+json") + public Response iiiFManifest(@PathParam("pid") String pid,@PathParam("source") String source) { + try { + ProxyItemHandler redirectHandler = findRedirectHandler(pid,source); + if (redirectHandler != null) { + return redirectHandler.iiifInfo(RequestMethodName.get, pid); + } else { + return Response.ok().build(); + } + } catch (WebApplicationException e) { + throw e; + } catch (Throwable e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new InternalErrorException(e.getMessage()); + } + } + + + @GET + @Produces("image/jpeg") + @Path("iiif/{pid}/{region}/{size}/{rotation}/default.jpg") + public Response tile(@PathParam("pid") String pid, + @PathParam("region") String region, + @PathParam("size") String size, + @PathParam("rotation") String rotation) { + + try { + + ProxyItemHandler redirectHandler = findRedirectHandler(pid,null); + if (redirectHandler != null) { + return redirectHandler.iiifTile(RequestMethodName.get, pid, region, size, rotation); + } else { + return Response.ok().build(); + } + + } catch (WebApplicationException e) { + throw e; + } catch (Throwable e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new InternalErrorException(e.getMessage()); + } + } + + @GET + @Produces("image/jpeg") + @Path("iiif/{source}/{pid}/{region}/{size}/{rotation}/default.jpg") + public Response tile(@PathParam("pid") String pid, + @PathParam("region") String region, + @PathParam("size") String size, + @PathParam("rotation") String rotation,@PathParam("source") String source) { + try { + ProxyItemHandler redirectHandler = findRedirectHandler(pid,source); + if (redirectHandler != null) { + return redirectHandler.iiifTile(RequestMethodName.get, pid, region, size, rotation); + } else { + return Response.ok().build(); + } + + } catch (WebApplicationException e) { + throw e; + } catch (Throwable e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new InternalErrorException(e.getMessage()); + } + } + } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java index e2d954ad73..5a97ad8c20 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java @@ -47,6 +47,11 @@ public ProxyItemHandler(Instances instances, User user, Client client, SolrAcces public abstract Response imagePreview(RequestMethodName method) throws ProxyHandlerException; public abstract Response imageThumb(RequestMethodName method) throws ProxyHandlerException; + + public abstract Response iiifInfo(RequestMethodName method, String pid) throws ProxyHandlerException; + public abstract Response iiifTile(RequestMethodName method, String pid, String region, String size, String rotation) throws ProxyHandlerException; + + public abstract Response zoomifyImageProperties(RequestMethodName method) throws ProxyHandlerException; public abstract Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHandlerException; diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java index 3ee68b6b86..e17ff5f9b1 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java @@ -156,4 +156,36 @@ protected JSONArray providedByLicense() { return retVal; } + + @Override + public Response iiifInfo(RequestMethodName method, String pid) throws ProxyHandlerException { + String baseurl = this.forwardUrl(); + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/iiif/" + this.pid + + "/info.json"; + + if (method == RequestMethodName.head) { + return buildForwardResponseHEAD(url); + } else { + return buildForwardResponseGET(url); + } + } + + + @Override + public Response iiifTile(RequestMethodName method, String pid, String region, String size, String rotation) + throws ProxyHandlerException { + + String baseurl = this.forwardUrl(); + String postfix = String.format("/%s/%s/%s/default.jpg", region,size, rotation); + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/iiif/" + this.pid+postfix; + + if (method == RequestMethodName.head) { + return buildForwardResponseHEAD(url); + } else { + return buildForwardResponseGET(url); + } + } + + + } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java index 475203571c..768c19e875 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java @@ -33,6 +33,7 @@ import cz.incad.kramerius.repository.KrameriusRepositoryApi.OwnRelationsMapping; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; +import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.ProxyItemHandler.RequestMethodName; import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.conf.KConfiguration; import cz.incad.kramerius.utils.pid.LexerException; @@ -105,6 +106,25 @@ public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHan return buildRedirectResponse(url); } + + + @Override + public Response iiifInfo(RequestMethodName method, String pid) throws ProxyHandlerException { + String baseurl = baseUrl(); + String formatted = String.format("iiif/%s/info.json", this.pid ); + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + formatted; + return buildRedirectResponse(url); + } + + @Override + public Response iiifTile(RequestMethodName method, String pid, String region, String size, String rotation) + throws ProxyHandlerException { + String baseurl = baseUrl(); + String postfix = String.format("%s/%s/%s/default.jpg", region, size, rotation); + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + postfix; + return buildRedirectResponse(url); + } + @Override public Response imageThumb(RequestMethodName method) throws ProxyHandlerException { String baseurl = baseUrl(); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java index e7a6b808dc..7387453640 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java @@ -181,5 +181,23 @@ public Response audioWAV() throws ProxyHandlerException { String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/client/v7.0/items/" + this.pid + "/audio/wav"; return buildRedirectResponse(url); } - + + + + @Override + public Response iiifInfo(RequestMethodName method, String pid) throws ProxyHandlerException { + // TODO Auto-generated method stub + return null; + } + + + + @Override + public Response iiifTile(RequestMethodName method, String pid, String region, String size, String rotation) + throws ProxyHandlerException { + // TODO Auto-generated method stub + return null; + } + + } diff --git a/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java b/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java index c465a2f6b5..56dfdbce8e 100644 --- a/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java +++ b/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java @@ -3,19 +3,31 @@ import com.google.inject.Inject; import com.google.inject.Provider; import com.google.inject.name.Named; +import com.sun.jersey.api.client.Client; + import cz.incad.Kramerius.AbstractImageServlet; import cz.incad.kramerius.FedoraAccess; import cz.incad.kramerius.ObjectPidsPath; import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.audio.AudioStreamForwardingHelper; +import cz.incad.kramerius.fedora.utils.CDKUtils; import cz.incad.kramerius.rest.api.k5.client.item.utils.IIIFUtils; +import cz.incad.kramerius.rest.apiNew.client.v60.ZoomifyHelper; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; +import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.ProxyItemHandler; import cz.incad.kramerius.security.RightsResolver; import cz.incad.kramerius.security.SecuredActions; import cz.incad.kramerius.security.User; import cz.incad.kramerius.statistics.StatisticsAccessLog; import cz.incad.kramerius.statistics.accesslogs.AggregatedAccessLogs; import cz.incad.kramerius.utils.FedoraUtils; +import cz.incad.kramerius.utils.IPAddressUtils; import cz.incad.kramerius.utils.RESTHelper; +import cz.incad.kramerius.utils.conf.KConfiguration; import cz.incad.kramerius.utils.imgs.KrameriusImageSupport; +import cz.incad.kramerius.utils.pid.LexerException; + import org.apache.commons.io.IOUtils; import org.json.JSONException; import org.json.JSONObject; @@ -23,12 +35,17 @@ import javax.servlet.ServletException; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; +import javax.ws.rs.core.Response; + import java.io.IOException; import java.io.InputStream; import java.io.PrintWriter; import java.net.HttpURLConnection; import java.net.URLDecoder; import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.List; +import java.util.Stack; import java.util.StringTokenizer; import java.util.logging.Level; @@ -38,11 +55,15 @@ public class IiifServlet extends AbstractImageServlet { @Inject - @Named("cachedSolrAccess") - private SolrAccess solrAccess; + Provider requestProvider; + /** + * Because of rights and licenses + */ @Inject - private RightsResolver rightsResolver; + @Named("new-index") + private SolrAccess solrAccess; + @Inject private Provider userProvider; @@ -52,6 +73,25 @@ public class IiifServlet extends AbstractImageServlet { private transient FedoraAccess fedoraAccess; + @Inject + ZoomifyHelper zoomifyHelper; + + @Inject + AudioStreamForwardingHelper audioHelper; + + @Inject + @Named("forward-client") + Provider clientProvider; + + + + @Inject + RightsResolver rightsResolver; + + @Inject + Instances instances; + + // @Inject // @Named("database") // private StatisticsAccessLog databaseAccessLog; @@ -67,57 +107,16 @@ public class IiifServlet extends AbstractImageServlet { @Override - protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException { + protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { try { - String requestURL = req.getRequestURL().toString(); - String zoomUrl = DeepZoomServlet.disectZoom(requestURL); - StringTokenizer tokenizer = new StringTokenizer(zoomUrl, "/"); - String pid = tokenizer.nextToken(); - - //unescape PID - pid = URLDecoder.decode(pid, "UTF-8"); - - - ObjectPidsPath[] paths = solrAccess.getPidPaths(pid); - boolean permited = false; - for (ObjectPidsPath pth : paths) { - permited = this.rightsResolver.isActionAllowed(userProvider.get(), SecuredActions.A_READ.getFormalName(), pid, null, pth.injectRepository()).flag(); - if (permited) break; - } - - if (permited) { - try { - String u = IIIFUtils.iiifImageEndpoint(pid, this.fedoraAccess); - StringBuilder url = new StringBuilder(u); - while (tokenizer.hasMoreTokens()) { - String nextToken = tokenizer.nextToken(); - url.append("/").append(nextToken); - if ("info.json".equals(nextToken)) { - reportAccess(pid); - resp.setContentType("application/ld+json"); - resp.setCharacterEncoding("UTF-8"); - HttpURLConnection con = (HttpURLConnection) RESTHelper.openConnection(url.toString(), "", ""); - InputStream inputStream = con.getInputStream(); - String json = IOUtils.toString(inputStream, Charset.defaultCharset()); - JSONObject object = new JSONObject(json); - String urlRequest = req.getRequestURL().toString(); - object.put("@id", urlRequest.substring(0, urlRequest.lastIndexOf('/'))); - PrintWriter out = resp.getWriter(); - out.print(object.toString()); - out.flush(); - return; - } - } - copyFromImageServer(url.toString(),resp); - } catch (JSONException e) { - LOGGER.log(Level.SEVERE, e.getMessage()); - } - } else { - resp.sendError(HttpServletResponse.SC_FORBIDDEN); - } + String pathInfo = req.getPathInfo(); + String redirectUrl = "/search/api/v7.0/client/items/iiif"+(!pathInfo.endsWith("/") ? "/":"") +pathInfo; + resp.sendRedirect(redirectUrl); } catch (IOException e) { LOGGER.severe(e.getMessage()); + resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); } + } @Override @@ -137,4 +136,7 @@ private void reportAccess(String pid) { LOGGER.log(Level.WARNING, "Can't write statistic records for " + pid, e); } } + + + } From 1d384c2369b73ee3be420d4c5aedf8523e32cf4b Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 19 Sep 2023 10:22:17 +0200 Subject: [PATCH 082/235] IIIF support for K5 --- .../incad/kramerius/rest/apiNew/client/v60/ItemsResource.java | 4 +++- search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java index 3f4e5f2ff0..1178a741f6 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java @@ -1215,7 +1215,8 @@ public Response getAudioWav(@PathParam("pid") String pid,@PathParam("source") St @Produces("application/ld+json") public Response iiiFManifest(@PathParam("pid") String pid) { try { - ProxyItemHandler redirectHandler = findRedirectHandler(pid,null); + LOGGER.info(String.format("Rendering iiif info.json (%s)", pid)); + ProxyItemHandler redirectHandler = findRedirectHandler(pid,null); if (redirectHandler != null) { return redirectHandler.iiifInfo(RequestMethodName.get, pid); } else { @@ -1234,6 +1235,7 @@ public Response iiiFManifest(@PathParam("pid") String pid) { @Produces("application/ld+json") public Response iiiFManifest(@PathParam("pid") String pid,@PathParam("source") String source) { try { + LOGGER.info(String.format("Rendering iiif info.json (%s,%s)", pid,source)); ProxyItemHandler redirectHandler = findRedirectHandler(pid,source); if (redirectHandler != null) { return redirectHandler.iiifInfo(RequestMethodName.get, pid); diff --git a/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java b/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java index 56dfdbce8e..1af0d86511 100644 --- a/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java +++ b/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java @@ -110,7 +110,7 @@ public class IiifServlet extends AbstractImageServlet { protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { try { String pathInfo = req.getPathInfo(); - String redirectUrl = "/search/api/v7.0/client/items/iiif"+(!pathInfo.endsWith("/") ? "/":"") +pathInfo; + String redirectUrl = "/search/api/v7.0/client/items/iiif"+(!pathInfo.startsWith("/") ? "/":"") +pathInfo; resp.sendRedirect(redirectUrl); } catch (IOException e) { LOGGER.severe(e.getMessage()); From c799c3feacbfd2dddca43aecebb5215808cd2fea Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 19 Sep 2023 10:38:26 +0200 Subject: [PATCH 083/235] IIIF suport - redirect fix --- search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java b/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java index 1af0d86511..e83a8d6053 100644 --- a/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java +++ b/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java @@ -110,7 +110,7 @@ public class IiifServlet extends AbstractImageServlet { protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { try { String pathInfo = req.getPathInfo(); - String redirectUrl = "/search/api/v7.0/client/items/iiif"+(!pathInfo.startsWith("/") ? "/":"") +pathInfo; + String redirectUrl = "/search/api/client/v7.0/items/iiif"+(!pathInfo.startsWith("/") ? "/":"") +pathInfo; resp.sendRedirect(redirectUrl); } catch (IOException e) { LOGGER.severe(e.getMessage()); From 1ad0b38abe1b2caa67eb37633cfcea2d4b69a657 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 19 Sep 2023 13:39:45 +0200 Subject: [PATCH 084/235] IIIF support; quality & format --- .../rest/apiNew/client/v60/ItemsResource.java | 20 ++++++++----- .../v60/redirection/ProxyHandlerSupport.java | 19 ++++++++++-- .../redirection/item/ProxyItemHandler.java | 2 +- .../redirection/item/V5ForwardHandler.java | 29 +++++++++++++++---- .../redirection/item/V5RedirectHandler.java | 4 +-- .../redirection/item/V7RedirectHandler.java | 7 +++-- 6 files changed, 61 insertions(+), 20 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java index 1178a741f6..d427300912 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java @@ -1253,17 +1253,21 @@ public Response iiiFManifest(@PathParam("pid") String pid,@PathParam("source") S @GET @Produces("image/jpeg") - @Path("iiif/{pid}/{region}/{size}/{rotation}/default.jpg") + @Path("iiif/{pid}/{region}/{size}/{rotation}/{qualityformat}") public Response tile(@PathParam("pid") String pid, @PathParam("region") String region, @PathParam("size") String size, - @PathParam("rotation") String rotation) { + @PathParam("rotation") String rotation, + + @PathParam("qualityformat") String qf + + ) { try { ProxyItemHandler redirectHandler = findRedirectHandler(pid,null); if (redirectHandler != null) { - return redirectHandler.iiifTile(RequestMethodName.get, pid, region, size, rotation); + return redirectHandler.iiifTile(RequestMethodName.get, pid, region, size, rotation,qf); } else { return Response.ok().build(); } @@ -1278,15 +1282,17 @@ public Response tile(@PathParam("pid") String pid, @GET @Produces("image/jpeg") - @Path("iiif/{source}/{pid}/{region}/{size}/{rotation}/default.jpg") - public Response tile(@PathParam("pid") String pid, + @Path("iiif/{source}/{pid}/{region}/{size}/{rotation}/{qualityformat}") + public Response tile(@PathParam("source") String source, @PathParam("pid") String pid, @PathParam("region") String region, @PathParam("size") String size, - @PathParam("rotation") String rotation,@PathParam("source") String source) { + @PathParam("rotation") String rotation, + @PathParam("qualityformat") String qf + ) { try { ProxyItemHandler redirectHandler = findRedirectHandler(pid,source); if (redirectHandler != null) { - return redirectHandler.iiifTile(RequestMethodName.get, pid, region, size, rotation); + return redirectHandler.iiifTile(RequestMethodName.get, pid, region, size, rotation,qf); } else { return Response.ok().build(); } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index fc9a9ed55c..4c17270bf1 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -8,10 +8,14 @@ import java.net.URL; import java.util.List; import java.util.Map; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import javax.activation.MimeType; +import javax.activation.MimeTypeParseException; import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.MultivaluedMap; import javax.ws.rs.core.Response; import javax.ws.rs.core.Response.ResponseBuilder; @@ -87,7 +91,11 @@ public ClientResponse forwardedResponse(String url) throws ProxyHandlerException } } - public Response buildForwardResponseGET(String url) throws ProxyHandlerException { + public Response buildForwardResponseGET(String url) throws ProxyHandlerException { + return buildForwardResponseGET(url, null); + } + + public Response buildForwardResponseGET(String url, String mimetype) throws ProxyHandlerException { WebResource.Builder b = buidFowrardResponse(url); ClientResponse response = b.get(ClientResponse.class); if (response.getStatus() == 200) { @@ -104,8 +112,13 @@ public void write(OutputStream output) throws IOException, WebApplicationExcepti } } }; - - ResponseBuilder respEntity = Response.status(200).entity(stream); + ResponseBuilder respEntity = null; + if (mimetype != null) { + //MimeType type = new MimeType(mimetype); + respEntity = Response.status(200).entity(stream).type(mimetype); + } else { + respEntity = Response.status(200).entity(stream); + } /* Disable header forward headers.keySet().forEach(key -> { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java index 5a97ad8c20..c1bda963a4 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java @@ -49,7 +49,7 @@ public ProxyItemHandler(Instances instances, User user, Client client, SolrAcces public abstract Response iiifInfo(RequestMethodName method, String pid) throws ProxyHandlerException; - public abstract Response iiifTile(RequestMethodName method, String pid, String region, String size, String rotation) throws ProxyHandlerException; + public abstract Response iiifTile(RequestMethodName method, String pid, String region, String size, String rotation, String qf) throws ProxyHandlerException; public abstract Response zoomifyImageProperties(RequestMethodName method) throws ProxyHandlerException; diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java index e17ff5f9b1..ae6e4f7da3 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java @@ -5,6 +5,8 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; +import java.util.HashMap; +import java.util.Map; import java.util.logging.Level; import java.util.logging.Logger; @@ -166,26 +168,43 @@ public Response iiifInfo(RequestMethodName method, String pid) throws ProxyHandl if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url); + return buildForwardResponseGET(url, null); } } + @Override - public Response iiifTile(RequestMethodName method, String pid, String region, String size, String rotation) + public Response iiifTile(RequestMethodName method, String pid, String region, String size, String rotation, String qf) throws ProxyHandlerException { + String defaultMime = IIIF_SUPPORTED_MIMETYPES.get("jpg"); + String baseurl = this.forwardUrl(); - String postfix = String.format("/%s/%s/%s/default.jpg", region,size, rotation); + String postfix = String.format("/%s/%s/%s/%s", region,size, rotation,qf); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/iiif/" + this.pid+postfix; - if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url); + String mime = defaultMime; + String[] splited = qf.split("\\."); + if (splited.length > 1) { + mime = IIIF_SUPPORTED_MIMETYPES.containsKey(splited[1]) ? IIIF_SUPPORTED_MIMETYPES.get(splited[1]) : defaultMime; + } + return buildForwardResponseGET(url, mime); } } + static Map IIIF_SUPPORTED_MIMETYPES = new HashMap<>(); + static { + IIIF_SUPPORTED_MIMETYPES.put("jpg", "image/jpeg"); + IIIF_SUPPORTED_MIMETYPES.put("tif", "image/tiff"); + IIIF_SUPPORTED_MIMETYPES.put("png", "image/png"); + IIIF_SUPPORTED_MIMETYPES.put("jp2", "image/jp2"); + IIIF_SUPPORTED_MIMETYPES.put("pdf", "application/pdf"); + IIIF_SUPPORTED_MIMETYPES.put("webp", "image/webp"); + } + } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java index 768c19e875..22752b17c6 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java @@ -117,10 +117,10 @@ public Response iiifInfo(RequestMethodName method, String pid) throws ProxyHandl } @Override - public Response iiifTile(RequestMethodName method, String pid, String region, String size, String rotation) + public Response iiifTile(RequestMethodName method, String pid, String region, String size, String rotation, String qf) throws ProxyHandlerException { String baseurl = baseUrl(); - String postfix = String.format("%s/%s/%s/default.jpg", region, size, rotation); + String postfix = String.format("%s/%s/%s/%s", region, size, rotation, qf); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + postfix; return buildRedirectResponse(url); } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java index 7387453640..2949932793 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java @@ -193,11 +193,14 @@ public Response iiifInfo(RequestMethodName method, String pid) throws ProxyHandl @Override - public Response iiifTile(RequestMethodName method, String pid, String region, String size, String rotation) - throws ProxyHandlerException { + public Response iiifTile(RequestMethodName method, String pid, String region, String size, String rotation, + String qf) throws ProxyHandlerException { // TODO Auto-generated method stub return null; } + + + } From 11bd8c6ec9516ecd9e2cbbe266c4b1cba99c1baf Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 26 Sep 2023 09:22:03 +0200 Subject: [PATCH 085/235] IIF Servlet & redirect --- .../rest/apiNew/client/v60/ItemsResource.java | 14 ++-- .../incad/Kramerius/imaging/IiifServlet.java | 77 ++++++++++++++++++- 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java index d427300912..28176e0349 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java @@ -1209,9 +1209,9 @@ public Response getAudioWav(@PathParam("pid") String pid,@PathParam("source") St } } - @GET - @Path("iiif/{pid}/info.json") + //@Path("iiif/{pid}/info.json") + @Path("{pid}/image/iiif/info.json") @Produces("application/ld+json") public Response iiiFManifest(@PathParam("pid") String pid) { try { @@ -1231,7 +1231,7 @@ public Response iiiFManifest(@PathParam("pid") String pid) { } @GET - @Path("iiif/{source}/{pid}/info.json") + @Path("{source}/{pid}/image/iiif/info.json") @Produces("application/ld+json") public Response iiiFManifest(@PathParam("pid") String pid,@PathParam("source") String source) { try { @@ -1253,14 +1253,13 @@ public Response iiiFManifest(@PathParam("pid") String pid,@PathParam("source") S @GET @Produces("image/jpeg") - @Path("iiif/{pid}/{region}/{size}/{rotation}/{qualityformat}") + //@Path("iiif/{pid}/{region}/{size}/{rotation}/{qualityformat}") + @Path("{pid}/image/iiif/{region}/{size}/{rotation}/{qualityformat}") public Response tile(@PathParam("pid") String pid, @PathParam("region") String region, @PathParam("size") String size, @PathParam("rotation") String rotation, - @PathParam("qualityformat") String qf - ) { try { @@ -1282,7 +1281,8 @@ public Response tile(@PathParam("pid") String pid, @GET @Produces("image/jpeg") - @Path("iiif/{source}/{pid}/{region}/{size}/{rotation}/{qualityformat}") + //@Path("iiif/{source}/{pid}/{region}/{size}/{rotation}/{qualityformat}") + @Path("{source}/{pid}/image/iiif/{region}/{size}/{rotation}/{qualityformat}") public Response tile(@PathParam("source") String source, @PathParam("pid") String pid, @PathParam("region") String region, @PathParam("size") String size, diff --git a/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java b/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java index e83a8d6053..9893c70e99 100644 --- a/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java +++ b/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java @@ -108,6 +108,46 @@ public class IiifServlet extends AbstractImageServlet { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { + + // search/iiif + // knav/uuid:xxxx + // nebo + // uuid:xxxx + + try { + String pathInfo = req.getPathInfo(); + if (pathInfo.contains("search/iiif/")) { + String afterServlet = afterServlet(pathInfo); + String[] split = afterServlet.split("/"); + + if (split.length >= 2) { + if (split[0].startsWith("uuid:")) { + + String startOfPid = pathInfo.substring(pathInfo.indexOf("uuid:")); + String pid = startOfPid.substring(0, startOfPid.indexOf("/")); + String end = pathInfo.substring(pathInfo.indexOf(pid)+pid.length()+1); + + String redirectUrl = String.format("/search/api/client/v7.0/items/%s/image/iiif/%s", pid, end); + resp.sendRedirect(redirectUrl); + + } else { + + String startOfPid = pathInfo.substring(pathInfo.indexOf("uuid:")); + String pid = startOfPid.substring(0, startOfPid.indexOf("/")); + String end = pathInfo.substring(pathInfo.indexOf(pid)+pid.length()+1); + String redirectUrl = String.format("/search/api/client/v7.0/items/%s/%s/image/iiif/%s", split[0], pid, end); + resp.sendRedirect(redirectUrl); + + } + } else { + resp.sendError(HttpServletResponse.SC_BAD_REQUEST); + } + } + } catch (IOException e) { + LOGGER.severe(e.getMessage()); + resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); + } + try { String pathInfo = req.getPathInfo(); String redirectUrl = "/search/api/client/v7.0/items/iiif"+(!pathInfo.startsWith("/") ? "/":"") +pathInfo; @@ -136,7 +176,42 @@ private void reportAccess(String pid) { LOGGER.log(Level.WARNING, "Can't write statistic records for " + pid, e); } } - + + +// public static void main(String[] args) { +// String pathInfo = "http://localhost:8080/search/iiif/uuid:xxxx/info.json"; +// String knavPathInfo = "http://localhost:8080/search/iiif/knav/uuid:xxxx/info.json"; +// +// String afterServlet = afterServlet(knavPathInfo); +// System.out.println("AFTER SERVLET :"+afterServlet); +// String[] split = afterServlet.split("/"); +// +// if (split[0].startsWith("uuid:")) { +// +// String startOfPid = pathInfo.substring(pathInfo.indexOf("uuid:")); +// String pid = startOfPid.substring(0, startOfPid.indexOf("/")); +// String end = pathInfo.substring(pathInfo.indexOf(pid)+pid.length()+1); +// +// String redirectUrl = String.format("/search/api/client/v7.0/items/%s/image/iiif/%s", pid, end); +// System.out.println(redirectUrl); +// +// +// } else { +// +// String startOfPid = pathInfo.substring(pathInfo.indexOf("uuid:")); +// String pid = startOfPid.substring(0, startOfPid.indexOf("/")); +// String end = pathInfo.substring(pathInfo.indexOf(pid)+pid.length()+1); +// String redirectUrl = String.format("/search/api/client/v7.0/items/%s/%s/image/iiif/%s", split[0], pid, end); +// +// //resp.sendRedirect(redirectUrl); +// System.out.println(redirectUrl); +// +// } +// } + private static String afterServlet(String pathInfo) { + String afterServlet = pathInfo.substring(pathInfo.indexOf("search/iiif/") + "search/iiif/".length()); + return afterServlet; + } } From 149d454e469f168e64733fb984df298160f4ea16 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 26 Sep 2023 09:43:45 +0200 Subject: [PATCH 086/235] FIx iiif servlet --- .../java/cz/incad/Kramerius/imaging/IiifServlet.java | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java b/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java index 9893c70e99..dc6478f8da 100644 --- a/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java +++ b/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java @@ -147,16 +147,6 @@ protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws Se LOGGER.severe(e.getMessage()); resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); } - - try { - String pathInfo = req.getPathInfo(); - String redirectUrl = "/search/api/client/v7.0/items/iiif"+(!pathInfo.startsWith("/") ? "/":"") +pathInfo; - resp.sendRedirect(redirectUrl); - } catch (IOException e) { - LOGGER.severe(e.getMessage()); - resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); - } - } @Override From 39a0f0ece84405dde21ce7ff6e3b989fd4e722b9 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 26 Sep 2023 12:41:54 +0200 Subject: [PATCH 087/235] Redirect --- .../incad/Kramerius/imaging/IiifServlet.java | 112 +++++++++--------- 1 file changed, 54 insertions(+), 58 deletions(-) diff --git a/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java b/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java index dc6478f8da..189450a9c3 100644 --- a/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java +++ b/search/src/java/cz/incad/Kramerius/imaging/IiifServlet.java @@ -109,44 +109,40 @@ public class IiifServlet extends AbstractImageServlet { @Override protected void doGet(HttpServletRequest req, HttpServletResponse resp) throws ServletException, IOException { - // search/iiif - // knav/uuid:xxxx - // nebo - // uuid:xxxx - try { String pathInfo = req.getPathInfo(); - if (pathInfo.contains("search/iiif/")) { - String afterServlet = afterServlet(pathInfo); - String[] split = afterServlet.split("/"); + if (pathInfo.startsWith("/")) { + pathInfo = pathInfo.substring(1); + } + String[] split = pathInfo.split("/"); + + if (split.length >= 2) { + if (split[0].startsWith("uuid:")) { + + String startOfPid = pathInfo.substring(pathInfo.indexOf("uuid:")); + String pid = startOfPid.substring(0, startOfPid.indexOf("/")); + String end = pathInfo.substring(pathInfo.indexOf(pid)+pid.length()+1); + + String redirectUrl = String.format("/search/api/client/v7.0/items/%s/image/iiif/%s", pid, end); + resp.sendRedirect(redirectUrl); - if (split.length >= 2) { - if (split[0].startsWith("uuid:")) { - - String startOfPid = pathInfo.substring(pathInfo.indexOf("uuid:")); - String pid = startOfPid.substring(0, startOfPid.indexOf("/")); - String end = pathInfo.substring(pathInfo.indexOf(pid)+pid.length()+1); - - String redirectUrl = String.format("/search/api/client/v7.0/items/%s/image/iiif/%s", pid, end); - resp.sendRedirect(redirectUrl); - - } else { - - String startOfPid = pathInfo.substring(pathInfo.indexOf("uuid:")); - String pid = startOfPid.substring(0, startOfPid.indexOf("/")); - String end = pathInfo.substring(pathInfo.indexOf(pid)+pid.length()+1); - String redirectUrl = String.format("/search/api/client/v7.0/items/%s/%s/image/iiif/%s", split[0], pid, end); - resp.sendRedirect(redirectUrl); - - } } else { - resp.sendError(HttpServletResponse.SC_BAD_REQUEST); + + String startOfPid = pathInfo.substring(pathInfo.indexOf("uuid:")); + String pid = startOfPid.substring(0, startOfPid.indexOf("/")); + String end = pathInfo.substring(pathInfo.indexOf(pid)+pid.length()+1); + String redirectUrl = String.format("/search/api/client/v7.0/items/%s/%s/image/iiif/%s", split[0], pid, end); + resp.sendRedirect(redirectUrl); + } + } else { + resp.sendError(HttpServletResponse.SC_BAD_REQUEST); } } catch (IOException e) { LOGGER.severe(e.getMessage()); resp.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR); } + } @Override @@ -169,36 +165,36 @@ private void reportAccess(String pid) { -// public static void main(String[] args) { -// String pathInfo = "http://localhost:8080/search/iiif/uuid:xxxx/info.json"; -// String knavPathInfo = "http://localhost:8080/search/iiif/knav/uuid:xxxx/info.json"; -// -// String afterServlet = afterServlet(knavPathInfo); -// System.out.println("AFTER SERVLET :"+afterServlet); -// String[] split = afterServlet.split("/"); -// -// if (split[0].startsWith("uuid:")) { -// -// String startOfPid = pathInfo.substring(pathInfo.indexOf("uuid:")); -// String pid = startOfPid.substring(0, startOfPid.indexOf("/")); -// String end = pathInfo.substring(pathInfo.indexOf(pid)+pid.length()+1); -// -// String redirectUrl = String.format("/search/api/client/v7.0/items/%s/image/iiif/%s", pid, end); -// System.out.println(redirectUrl); -// -// -// } else { -// -// String startOfPid = pathInfo.substring(pathInfo.indexOf("uuid:")); -// String pid = startOfPid.substring(0, startOfPid.indexOf("/")); -// String end = pathInfo.substring(pathInfo.indexOf(pid)+pid.length()+1); -// String redirectUrl = String.format("/search/api/client/v7.0/items/%s/%s/image/iiif/%s", split[0], pid, end); -// -// //resp.sendRedirect(redirectUrl); -// System.out.println(redirectUrl); -// -// } -// } + public static void main(String[] args) { + String pathInfo = "iiif/knav/uuid:765f3df0-1345-4439-92f7-04344b73880a/info.json"; + String knavPathInfo = "http://localhost:8080/search/iiif/knav/uuid:xxxx/info.json"; + + String afterServlet = afterServlet(pathInfo); + System.out.println("AFTER SERVLET :"+afterServlet); + String[] split = afterServlet.split("/"); + + if (split[0].startsWith("uuid:")) { + + String startOfPid = pathInfo.substring(pathInfo.indexOf("uuid:")); + String pid = startOfPid.substring(0, startOfPid.indexOf("/")); + String end = pathInfo.substring(pathInfo.indexOf(pid)+pid.length()+1); + + String redirectUrl = String.format("/search/api/client/v7.0/items/%s/image/iiif/%s", pid, end); + System.out.println(redirectUrl); + + + } else { + + String startOfPid = pathInfo.substring(pathInfo.indexOf("uuid:")); + String pid = startOfPid.substring(0, startOfPid.indexOf("/")); + String end = pathInfo.substring(pathInfo.indexOf(pid)+pid.length()+1); + String redirectUrl = String.format("/search/api/client/v7.0/items/%s/%s/image/iiif/%s", split[0], pid, end); + + //resp.sendRedirect(redirectUrl); + System.out.println(redirectUrl); + + } + } private static String afterServlet(String pathInfo) { String afterServlet = pathInfo.substring(pathInfo.indexOf("search/iiif/") + "search/iiif/".length()); From 34c27e881657c60a9268676baca3ae518d805afa Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 3 Oct 2023 10:53:17 +0200 Subject: [PATCH 088/235] Issue #42 --- .../transform/K7SourceToDestTransform.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java index 13575b2def..fb0f613b1a 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java @@ -361,7 +361,39 @@ public boolean acceptElement(Element element) { DateInfo dateInfo = dateExtractor.extractFromString(textContent.toString(), pid); appendDateFields(destDocument, destDocElem, dateInfo, consumer ); } + + + // Issue 42 / https://github.com/ceskaexpedice/ceska-digitalni-knihovna/issues/42 + if (document.containsKey("model")) { + String model = document.get("model").get(0); + if (model.equals("page")) { + Element details = XMLUtils.findElement(sourceDocElm, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String nodeName = element.getNodeName(); + if (nodeName.equals("arr")) { + String nameAttr = element.getAttribute("name"); + if (nameAttr != null) return "details".equals(nameAttr); + } + return false; + } + }); + + if (details != null) { + List elements = XMLUtils.getElements(details); + if (elements != null && elements.size() > 0) { + String textContent = elements.get(0).getTextContent(); + String[] split = textContent.split("##"); + if (split.length> 1) { + field(destDocument, destDocElem,split[0], "page.type",consumer); + } + } + } + } + } } + + } private void field(Document destDocument, Element destDocElem, String value, String targetName, CopyReplicateConsumer consumer) { From 9fff6263b62dfce0d32117a56daebce0006b035f Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 9 Oct 2023 11:59:33 +0200 Subject: [PATCH 089/235] Issue https://github.com/ceskaexpedice/ceska-digitalni-knihovna/issues/51 --- .../services/KubernetesProcessImpl.java | 5 + .../services/ParallelProcessImpl.java | 148 +++++++++++++- .../services/SupportedLibraries.java | 54 ++++++ .../transform/BasicSourceToDestTransform.java | 150 +-------------- .../transform/K7SourceToDestTransform.java | 28 ++- .../replicate/AbstractReplicateWorker.java | 76 +++++--- .../workers/replicate/BatchUtils.java | 26 --- .../replicate/copy/CopyReplicateConsumer.java | 4 +- .../replicate/copy/CopyReplicateWorker.java | 181 ++++++++++++------ .../cz/incad/kramerius/services/registr | 1 + .../replicate/configurations/default_k5.xml | 6 +- .../sliced/nkp/nkp_sliced_1.xml | 93 +++++++++ .../sliced/nkp/nkp_sliced_10.xml | 93 +++++++++ .../sliced/nkp/nkp_sliced_11.xml | 93 +++++++++ .../sliced/nkp/nkp_sliced_12.xml | 93 +++++++++ .../sliced/nkp/nkp_sliced_2.xml | 93 +++++++++ .../sliced/nkp/nkp_sliced_3.xml | 93 +++++++++ .../sliced/nkp/nkp_sliced_4.xml | 93 +++++++++ .../sliced/nkp/nkp_sliced_5.xml | 93 +++++++++ .../sliced/nkp/nkp_sliced_6.xml | 93 +++++++++ .../sliced/nkp/nkp_sliced_7.xml | 93 +++++++++ .../sliced/nkp/nkp_sliced_8.xml | 93 +++++++++ .../sliced/nkp/nkp_sliced_9.xml | 93 +++++++++ .../services/ParallelProcessImplTest.java | 122 ++++++++++++ 24 files changed, 1656 insertions(+), 261 deletions(-) create mode 100644 processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/SupportedLibraries.java create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/registr create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_1.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_10.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_11.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_12.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_2.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_3.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_4.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_5.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_6.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_7.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_8.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_9.xml create mode 100644 processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/ParallelProcessImplTest.java diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java index 37415e9af3..80487b65f8 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java @@ -8,6 +8,7 @@ import org.apache.commons.io.IOUtils; import org.xml.sax.SAXException; + import javax.xml.parsers.ParserConfigurationException; import java.io.File; import java.io.IOException; @@ -15,6 +16,7 @@ import java.io.InputStreamReader; import java.util.HashMap; import java.util.Map; +import java.util.TimeZone; import java.util.logging.Logger; /** @@ -39,6 +41,9 @@ public class KubernetesProcessImpl { public static final Logger LOGGER = Logger.getLogger(KubernetesProcessImpl.class.getName()); public static void main(String[] args) throws IOException, MigrateSolrIndexException, IllegalAccessException, InstantiationException, SAXException, ParserConfigurationException, NoSuchMethodException, ClassNotFoundException { + + TimeZone.setDefault(TimeZone.getTimeZone("Europe/Prague")); + Map env = System.getenv(); if (env.containsKey(CONFIG_SOURCE) || args.length > 0) { String configSource = env.containsKey(CONFIG_SOURCE) ? env.get(CONFIG_SOURCE) : args[0]; diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java index 04d5b69a72..878e2a5e3e 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java @@ -18,10 +18,20 @@ import java.io.File; import java.io.FileInputStream; import java.io.IOException; +import java.text.SimpleDateFormat; +import java.time.Duration; +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.LocalTime; +import java.time.format.DateTimeFormatter; +import java.time.temporal.TemporalUnit; import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; import java.util.List; import java.util.concurrent.BrokenBarrierException; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; @@ -67,7 +77,124 @@ protected Client buildClient() { return Client.create(cc); } - private void startWorkers(List worksWhasHasToBeDone) throws BrokenBarrierException, InterruptedException { + + public static boolean isInWorkingTime( String startTime, String endTime) { + try { + LocalDate now = LocalDate.now(); + LocalDateTime current = LocalDateTime.now(); + + return isWorkingTimeImpl(startTime, endTime, now, current); + } catch (Exception e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + return false; + } + } + + public static boolean isWorkingTimeImpl(String startTime,String endTime, LocalDate now, LocalDateTime current) { + + LocalTime startTimeLT = LocalTime.parse(startTime, DateTimeFormatter.ofPattern("H:mm")); + LocalDateTime startDateTimeLT = startTimeLT.atDate(now); + + LocalTime endTimeLT = LocalTime.parse(endTime, DateTimeFormatter.ofPattern("H:mm")); + LocalDateTime endDateTimeLT = endTimeLT.atDate(now); + + if (startDateTimeLT.isAfter(endDateTimeLT)) { + if (current.isAfter(startDateTimeLT.minusDays(1)) && current.isBefore(endDateTimeLT)) { + return true; + } else if (current.isAfter(startDateTimeLT) && current.isBefore(endDateTimeLT.plusDays(1))) { + return true; + } + } else { + if ( current.isAfter(startDateTimeLT) && current.isBefore(endDateTimeLT)) { + return true; + } else { + return false; + } + } + + return false; + } + + public static void waitUntilStartWorkingTime(String startTime, String endTime) { + try { + LocalDate now = LocalDate.now(); + LocalDateTime current = LocalDateTime.now(); + + long time = waitUntilStartWorkingTimeImpl(startTime, endTime, now, current); + Thread.sleep(time); + + } catch (Exception e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } + } + + public static long waitUntilStartWorkingTimeImpl(String startTime, String endTime, LocalDate now, + LocalDateTime current) { + LocalTime startTimeLT = LocalTime.parse(startTime, DateTimeFormatter.ofPattern("H:mm")); + LocalDateTime startDateTimeLT = startTimeLT.atDate(now); + + LocalTime endTimeLT = LocalTime.parse(endTime, DateTimeFormatter.ofPattern("H:mm")); + LocalDateTime endDateTimeLT = endTimeLT.atDate(now); + + if (startDateTimeLT.isAfter(endDateTimeLT)) { + if (current.isAfter(startDateTimeLT.minusDays(1)) && current.isBefore(endDateTimeLT)) { + startDateTimeLT = startDateTimeLT.minusDays(1); + } else if (current.isAfter(startDateTimeLT) && current.isBefore(endDateTimeLT.plusDays(1))) { + endDateTimeLT = endDateTimeLT.plusDays(1); + } + } + + + + Duration duration = Duration.between(current, startDateTimeLT); + if (!duration.isNegative()) { + long millis = duration.toMillis(); + + waitingInfo(startDateTimeLT, millis); + + return millis; + //Thread.sleep(millis); + } else { + startDateTimeLT = startDateTimeLT.plusDays(1); + endDateTimeLT = endDateTimeLT.plusDays(1); + LOGGER.info("I'm shifting day to :"+startDateTimeLT); + duration = Duration.between(current, startDateTimeLT); + long millis = duration.toMillis(); + + waitingInfo(startDateTimeLT, millis); + + return millis; + //Thread.sleep(millis); + + //LOGGER.log(Level.SEVERE, "Negative waiting"); + } + } + + private static void waitingInfo(LocalDateTime startDateTimeLT, long millis) { + long hours = millis / 3600000; // 3600000 ms v hodině + long minutes = (millis % 3600000) / 60000; // 60000 ms v minutě + long seconds = ((millis % 3600000) % 60000) / 1000; // 1000 ms v sekundě + + LOGGER.info("The date is "+startDateTimeLT+". The calculated wait time is : " + hours + " hours, " + minutes + " minutes a " + seconds + " seconds."); + } + + private void startWorkers(List worksWhasHasToBeDone, String workingtime) throws BrokenBarrierException, InterruptedException { + if (workingtime != null && workingtime.contains("-")) { + String[] intervalParts = workingtime.split("-"); + String startTime = intervalParts[0]; + String endTime = intervalParts[1]; + + while (true) { + if (!isInWorkingTime( startTime, endTime)) { + //LOGGER.info(String.format("No harvesting - waiting for end of interval %s", workingtime)); + waitUntilStartWorkingTime(startTime, endTime); + } else { + break; + } + } + } + + // check if sleep hours CyclicBarrier barrier = new CyclicBarrier(worksWhasHasToBeDone.size()+1); worksWhasHasToBeDone.stream().forEach(th->{ th.setBarrier(barrier); @@ -89,6 +216,13 @@ public void migrate(File config) throws MigrateSolrIndexException, IllegalAccess timestamp = timestampElm.getTextContent(); } + Element workingTimeElm = XMLUtils.findElement(document.getDocumentElement(),"workingtime"); + final AtomicReference workingTime = new AtomicReference<>(); + if (workingTimeElm != null) { + workingTime.set(workingTimeElm.getTextContent()); + //workingtime = workingTimeElm.getTextContent(); + } + Element sourceNameElm = XMLUtils.findElement(document.getDocumentElement(), "source-name"); if (sourceNameElm != null) { this.sourceName = sourceNameElm.getTextContent(); @@ -124,9 +258,9 @@ public void migrate(File config) throws MigrateSolrIndexException, IllegalAccess final List worksWhatHasToBeDone = new ArrayList<>(); try { this.iterator.iterate(this.client, (List idents)->{ - addNewWorkToWorkers(this.iterator, worksWhatHasToBeDone, idents); + addNewWorkToWorkers(this.iterator, worksWhatHasToBeDone, idents, workingTime.get()); }, ()-> { - finishRestWorkers(worksWhatHasToBeDone); + finishRestWorkers(worksWhatHasToBeDone,workingTime.get()); }); } catch (Exception e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); @@ -142,11 +276,11 @@ private void initialize(Element iteration) { } // musi zustat tady - private void addNewWorkToWorkers(ProcessIterator processIterator, List worksWhatHasToBeDone, List identifiers) { + private void addNewWorkToWorkers(ProcessIterator processIterator, List worksWhatHasToBeDone, List identifiers, String workingtime) { try { worksWhatHasToBeDone.add(createWorker(processIterator, this.workerElem, identifiers)); if (worksWhatHasToBeDone.size() >= threads) { - startWorkers(worksWhatHasToBeDone); + startWorkers(worksWhatHasToBeDone, workingtime); worksWhatHasToBeDone.clear(); } } catch ( BrokenBarrierException | InterruptedException e) { @@ -164,10 +298,10 @@ private Worker createWorker(ProcessIterator iteratorInstance, Element workerElm, } } - private void finishRestWorkers(List worksWhatHasToBeDone) { + private void finishRestWorkers(List worksWhatHasToBeDone,String workingtime) { try { if (!worksWhatHasToBeDone.isEmpty()) { - startWorkers(worksWhatHasToBeDone); + startWorkers(worksWhatHasToBeDone, workingtime); } } catch (BrokenBarrierException | InterruptedException e) { LOGGER.log(Level.SEVERE,e.getMessage(),e); diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/SupportedLibraries.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/SupportedLibraries.java new file mode 100644 index 0000000000..ba01736df1 --- /dev/null +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/SupportedLibraries.java @@ -0,0 +1,54 @@ +package cz.incad.kramerius.services; + +import java.io.IOException; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +public class SupportedLibraries { + + public static final Logger LOGGER = Logger.getLogger(SupportedLibraries.class.getName()); + + private List codes = new ArrayList<>(); + + public SupportedLibraries() { + try { + InputStream resourceAsStream = this.getClass().getResourceAsStream("registr"); + String str = IOUtils.toString(resourceAsStream, "UTF-8"); + JSONArray jsonObj = new JSONArray(str); + for (int i = 0; i < jsonObj.length(); i++) { + JSONObject libObject = jsonObj.getJSONObject(i); + String code = libObject.optString("code"); + if (code != null) codes.add(code); + + } + } catch (JSONException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } catch (IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } + } + + public List getCodes() { + return codes; + } + + public Pair divideLibraryAndLicense(String cdklicense) { + for (String code : codes) { + if (cdklicense.startsWith(code+"_")) { + int index = cdklicense.indexOf(code+"_")+(code).length(); + return Pair.of(cdklicense.substring(0, index), cdklicense.substring(index+1)); + } + } + return null; + } + +} diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/BasicSourceToDestTransform.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/BasicSourceToDestTransform.java index bda343dae2..f4ae949539 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/BasicSourceToDestTransform.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/BasicSourceToDestTransform.java @@ -21,31 +21,6 @@ //TODO: Change it in future; copied from K5 cdk download public class BasicSourceToDestTransform extends SourceToDestTransform{ - // text is copied but not for PDF; uuugrrrr !!! Terrible - public static final List EXCEPTION_FIELDS = Arrays.asList("text"); - // copied - public static final List COPIED_FIELDS = Arrays.asList("title", "search_title","facet_autor","search_autor"); - // copied but identified by postfix - public static final List COPIED_POSTFIXES = Arrays.asList("_lemmatized","_lemmatized_ascii","_lemmatized_nostopwords"); - - static boolean exceptionField(String attributeName) { - if (EXCEPTION_FIELDS.contains(attributeName)) { - return true; - } - return false; - } - - static boolean nonCopiingField(String attributeName) { - if (COPIED_FIELDS.contains(attributeName)) { - return true; - } - for (String postfix: - COPIED_POSTFIXES) { - if (attributeName.endsWith(postfix)) return true; - } - return false; - } - /** find element by attribute */ @@ -64,7 +39,7 @@ public boolean acceptElement(Element element) { public static void simpleValue(String pid, Document feedDoc, Element feedDocElm, Node node, String derivedName, boolean dontCareAboutNonCopiingFields, CopyReplicateConsumer consumer) { //String attributeName = ((Element)node).getAttribute("name"); String attributeName = derivedName != null ? derivedName : ((Element)node).getAttribute("name"); - if (dontCareAboutNonCopiingFields || !nonCopiingField(attributeName)) { + if (dontCareAboutNonCopiingFields) { Element strElm = feedDoc.createElement("field"); strElm.setAttribute("name", attributeName); @@ -84,37 +59,12 @@ public static void simpleValue(String pid, Document feedDoc, Element feedDocElm, public static void arrayValue(String pid, Element sourceDocElement, Document feedDoc, Element feedDocElement, Node node, CopyReplicateConsumer consumer) { String attributeName = ((Element) node).getAttribute("name"); - if (!nonCopiingField(attributeName)) { - if (exceptionField(attributeName) && pid.contains("/@")) { - NodeList childNodes = node.getChildNodes(); - for (int i = 0,ll=childNodes.getLength(); i < ll; i++) { - Node n = childNodes.item(i); - if (n.getNodeType() == Node.ELEMENT_NODE) { - // exception again !!! uuugrrrr !!; - - // bug in pdf; text is filled directly to text field although text is copied field - // first we have to find text_ocr, if it doesn't exist, copy whole text to text, text_lemmatized, text_lemmatized_ascii and text_lemmatized_nostopwords - Element textOcr = findByAttribute(sourceDocElement, "text_ocr"); - if (textOcr == null) { - - simpleValue(pid, feedDoc,feedDocElement, n, attributeName, false, consumer); - simpleValue(pid, feedDoc, feedDocElement, n,"text_lemmatized", true, consumer); - simpleValue(pid, feedDoc, feedDocElement, n,"text_lemmatized_ascii", true, consumer); - simpleValue(pid, feedDoc, feedDocElement, n,"text_lemmatized_nostopwords", true, consumer); - - } - } - } - } else if (!exceptionField(attributeName)) { - NodeList childNodes = node.getChildNodes(); - for (int i = 0,ll=childNodes.getLength(); i < ll; i++) { - Node n = childNodes.item(i); - if (n.getNodeType() == Node.ELEMENT_NODE) { - //simpleValue(feedDoc,feedDocElement, n, false); - simpleValue(pid, feedDoc,feedDocElement, n, attributeName, false, consumer); - - } - } + NodeList childNodes = node.getChildNodes(); + for (int i = 0,ll=childNodes.getLength(); i < ll; i++) { + Node n = childNodes.item(i); + if (n.getNodeType() == Node.ELEMENT_NODE) { + //simpleValue(feedDoc,feedDocElement, n, false); + simpleValue(pid, feedDoc,feedDocElement, n, attributeName, false, consumer); } } } @@ -136,7 +86,7 @@ public boolean acceptElement(Element element) { } else return ""; } - /** transforming fields in k5 index; it doesn't apply if an index is K7 */ + /** */ public void transform(Element sourceDocElm, Document destDocument, Element destDocElem, CopyReplicateConsumer consumer) { String pid = pid(sourceDocElm); if (sourceDocElm.getNodeName().equals("doc")) { @@ -152,93 +102,9 @@ public void transform(Element sourceDocElm, Document destDocument, Element destD } } } - - browseAuthorsAndTitles(sourceDocElm, destDocument, destDocElem); } } - // special not stored fields browse_autor, browse_title - public static void browseAuthorsAndTitles(Element sourceDocElm,Document ndoc, Element docElm) { - try { - UTFSort utf_sort = new UTFSort(); - //FedoraOperations operations = new FedoraOperations(); - - // browse author -- skip - Element browseAuthorInSource = XMLUtils.findElement(sourceDocElm, new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - return element.getAttribute("name").equals("browse_autor"); - } - }); - // browse title doens't exist - if (browseAuthorInSource == null) { - Element dcCreators = XMLUtils.findElement(sourceDocElm, new XMLUtils.ElementsFilter() { - - @Override - public boolean acceptElement(Element e) { - String attribute = e.getAttribute("name"); - return attribute.equals("dc.creator"); - } - }); - if (dcCreators != null) { - List dcCreatorsStrings = XMLUtils.getElements(dcCreators); - for (Element author : dcCreatorsStrings) { - //## - String textContent = author.getTextContent(); - String prepared = utf_sort.translate(textContent)+"##"+textContent; - //String prepared = operations.prepareCzech(textContent)+"##"+textContent; - Element strElm = ndoc.createElement("field"); - strElm.setAttribute("name", "browse_autor"); - docElm.appendChild(strElm); - strElm.setTextContent(prepared); - } - } - - } - - - // browse author -- skip - Element browseTitleInSource = XMLUtils.findElement(sourceDocElm, new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - return element.getAttribute("name").equals("browse_title"); - } - }); - - if (browseTitleInSource == null) { - Element model = XMLUtils.findElement(sourceDocElm, new XMLUtils.ElementsFilter() { - - @Override - public boolean acceptElement(Element e) { - String attribute = e.getAttribute("name"); - return attribute.equals("fedora.model"); - } - }); - - - Element dcTitle = XMLUtils.findElement(sourceDocElm, new XMLUtils.ElementsFilter() { - - @Override - public boolean acceptElement(Element e) { - String attribute = e.getAttribute("name"); - return attribute.equals("dc.title"); - } - }); - - if (dcTitle != null && model != null && Arrays.asList(KConfiguration.getInstance().getConfiguration().getStringArray("indexer.browseModels")).contains(model.getTextContent().trim())) { - Element strElm = ndoc.createElement("field"); - strElm.setAttribute("name", "browse_title"); - docElm.appendChild(strElm); - String textContent = dcTitle.getTextContent(); - String prepared = utf_sort.translate(textContent)+"##"+textContent; - strElm.setTextContent(prepared); - } - } - } catch (Exception e) { - BatchUtils.LOGGER.log(Level.SEVERE,e.getMessage(), e); - throw new RuntimeException(e.getMessage()); - } - } @Override public String getField(String fieldId) { diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java index fb0f613b1a..41e659703b 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java @@ -385,17 +385,43 @@ public boolean acceptElement(Element element) { String textContent = elements.get(0).getTextContent(); String[] split = textContent.split("##"); if (split.length> 1) { - field(destDocument, destDocElem,split[0], "page.type",consumer); + field(destDocument, destDocElem,split[1], "page.type",consumer); } } } } } + + if (document.containsKey("accessibility") && document.containsKey("model")) { + String accessibility = document.get("accessibility").get(0); + String model = document.get("model").get(0); + + translateAccessibilityToLicense(model, accessibility, destDocument, destDocElem, consumer); + } } + } + private void translateAccessibilityToLicense(String model, String accessibility, Document destDocument, Element destDocElem, CopyReplicateConsumer consumer) { + + String license = "public".equals(accessibility) ? "public" : "onsite"; + List markContainsLicenses = Arrays.asList("monographunit","periodicalvolume"); + + List licenses = Arrays.asList("monograph","monographunit", "periodicalvolume","map","sheetmusic"); + List licenses_of_ancestors = Arrays.asList("page","supplement","periodicalitem"); + if (licenses.contains(model)) { + field(destDocument, destDocElem,license, "licenses",consumer); + } else if (licenses_of_ancestors.contains(model)) { + field(destDocument, destDocElem,license, "licenses_of_ancestors",consumer); + } + + if (markContainsLicenses.contains(model)) { + field(destDocument, destDocElem,license, "cdk.k5.marked_for_contains_licenses",consumer); + } + field(destDocument, destDocElem,license, "cdk.k5.license.translated",consumer); } + private void field(Document destDocument, Element destDocElem, String value, String targetName, CopyReplicateConsumer consumer) { Element strElm = destDocument.createElement("field"); strElm.setAttribute("name", targetName); diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java index f21d18de9a..f57eb05005 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java @@ -3,6 +3,7 @@ import java.io.IOException; import java.net.URLEncoder; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -61,15 +62,12 @@ protected ReplicateContext findPidsAlreadyIndexed(List subpids, SourceTo String reduce = subpids.stream().map(it -> { return '"' + it + '"'; }).collect(Collectors.joining(" OR ")); + - // field list je pid + collection + rootpid + dalsi field list - //String fieldlist = this.transform.getField(idIdentifier) + " " + collectionField +" cdk.leader cdk.licenses cdk.collection"; - String fieldlist = "pid " + collectionField +" cdk.leader cdk.licenses cdk.collection"; + List computedFields = Arrays.asList("cdk.licenses", "cdk.licenses_of_ancestors cdk.contains_licenses"); + String fieldlist = "pid " + collectionField +" cdk.leader cdk.collection "+computedFields.stream().collect(Collectors.joining(" ")); if (compositeId) { fieldlist = fieldlist + " " + " root.pid compositeId"; -// if (!idIdentifier.equals(childOfComposite)) { -// fieldlist = fieldlist + " " + this.transform.getField(this.childOfComposite); -// } } String query = "?q=" + "pid" + ":(" + URLEncoder.encode(reduce, "UTF-8") @@ -82,16 +80,32 @@ protected ReplicateContext findPidsAlreadyIndexed(List subpids, SourceTo }); List docs = XMLUtils.getElements(resultElem); + List> list = new ArrayList<>(); docs.stream().forEach(d -> { + List simpleFields = Arrays.asList("str","date","int"); Map map = new HashMap<>(); - - Element pid = XMLUtils.findElement(d, e -> { - return e.getAttribute("name").equals("pid"); + List fields = XMLUtils.getElements(d); + fields.stream().forEach(f-> { + if (simpleFields.contains(f.getNodeName())) { + String name = f.getAttribute("name"); + map.put(name, f.getTextContent()); + } else { + String name = f.getAttribute("name"); + List elements = XMLUtils.getElements(f); + List contents = elements.stream().map(Element::getTextContent).collect(Collectors.toList()); + map.put(name, contents); + } }); - if (pid != null) { - map.put("pid", pid.getTextContent()); - } + + +// Element pid = XMLUtils.findElement(d, e -> { +// return e.getAttribute("name").equals("pid"); +// }); +// +// if (pid != null) { +// map.put("pid", pid.getTextContent()); +// } Element collection = XMLUtils.findElement(d, e -> { return e.getAttribute("name").equals(collectionField); @@ -101,24 +115,16 @@ protected ReplicateContext findPidsAlreadyIndexed(List subpids, SourceTo map.put(collectionField, collection.getTextContent()); } - Element cdkLeader = XMLUtils.findElement(d, e -> { - return e.getAttribute("name").equals("cdk.leader"); - }); - - if (cdkLeader != null) { - map.put("cdk.leader", cdkLeader.getTextContent()); - } - - Element cdkLicenses = XMLUtils.findElement(d, e -> { - return e.getAttribute("name").equals("cdk.licenses"); - }); - - if (cdkLicenses != null) { - List licenses = XMLUtils.getElements(cdkLicenses).stream().map(Element::getTextContent).collect(Collectors.toList()); - map.put("cdk.licenses", licenses); - } - +// Element cdkLeader = XMLUtils.findElement(d, e -> { +// return e.getAttribute("name").equals("cdk.leader"); +// }); +// +// if (cdkLeader != null) { +// map.put("cdk.leader", cdkLeader.getTextContent()); +// } + // computed fields + computedFields.stream().forEach(it-> computedField(d, map,it)); if (compositeId) { Element compositeRoot = XMLUtils.findElement(d, e -> { @@ -152,6 +158,18 @@ protected ReplicateContext findPidsAlreadyIndexed(List subpids, SourceTo return new ReplicateContext(list, notindexed); } + private void computedField(Element d, Map map, String fieldName) { + Element cdkLicenses = XMLUtils.findElement(d, e -> { + return e.getAttribute("name").equals(fieldName); + //return e.getAttribute("name").equals("cdk.licenses"); + }); + + if (cdkLicenses != null) { + List licenses = XMLUtils.getElements(cdkLicenses).stream().map(Element::getTextContent).collect(Collectors.toList()); + map.put(fieldName, licenses); + } + } + protected void config(Element workerElm) { Element destinationElm = XMLUtils.findElement(workerElm, "destination"); if (destinationElm != null) { diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/BatchUtils.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/BatchUtils.java index cefb1e7a7e..447d4a266f 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/BatchUtils.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/BatchUtils.java @@ -36,15 +36,6 @@ public boolean acceptElement(Element elm) { return (elm.getNodeName().equals("doc")); } }); - - StringWriter writer = new StringWriter(); - try { - XMLUtils.print(resultElem, writer); - } catch (TransformerException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - for (int i = 0; i < docs.size(); i++) { Element destDocElement = destBatch.createElement("doc"); @@ -53,15 +44,6 @@ public boolean acceptElement(Element elm) { // basic transform srcTransform.transform(sourceDocElm, destBatch, destDocElement,consumer); - writer = new StringWriter(); - try { - XMLUtils.print(destDocElement, writer); - } catch (TransformerException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - - // composite id is not supported if (compositeId && root != null && child != null) { @@ -98,14 +80,6 @@ public boolean acceptElement(Element element) { } }); - writer = new StringWriter(); - try { - XMLUtils.print(destDocElement, writer); - } catch (TransformerException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - consumer.changeDocument(rootPidElm.getTextContent(),pidElm.getTextContent(), destDocElement); } return destBatch; diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateConsumer.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateConsumer.java index dc38fe573b..e21bcd0bd8 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateConsumer.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateConsumer.java @@ -5,7 +5,9 @@ public interface CopyReplicateConsumer { public enum ModifyFieldResult { - edit,delete,none; + edit,delete,none, + + calculated; } public ModifyFieldResult modifyField(Element field); diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java index 4bcfc24f43..1235f5d632 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java @@ -2,6 +2,7 @@ import com.sun.jersey.api.client.Client; import cz.incad.kramerius.service.MigrateSolrIndexException; +import cz.incad.kramerius.services.SupportedLibraries; import cz.incad.kramerius.services.Worker; import cz.incad.kramerius.services.iterators.IterationItem; import cz.incad.kramerius.services.utils.SolrUtils; @@ -22,9 +23,12 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import java.util.logging.Level; import java.util.logging.Logger; @@ -35,6 +39,9 @@ */ public class CopyReplicateWorker extends AbstractReplicateWorker { + //public static List LIBRARIES = Arrays.asList("nkp","blbec_k5"); + public static SupportedLibraries supportedLibraries = new SupportedLibraries(); + public static Logger LOGGER = Logger.getLogger(CopyReplicateWorker.class.getName()); public CopyReplicateWorker(String sourceName, Element workerElm, Client client, List pids) { @@ -73,25 +80,60 @@ public ModifyFieldResult modifyField(Element field) { @Override public void changeDocument(String rootPid, String pid,Element doc) { - List foundElements = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { + + List licensesOfAncestors = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { @Override public boolean acceptElement(Element element) { String attribute = element.getAttribute("name"); - return "licenses".equals(attribute); + return "licenses_of_ancestors".equals(attribute); } - }); + }).stream().map(Element::getTextContent).collect(Collectors.toList()); - if (!foundElements.isEmpty()) { - for (Element fElm : foundElements) { - Document document = fElm.getOwnerDocument(); - Element cdkLicenses = document.createElement("field"); - cdkLicenses.setAttribute("name", "cdk.licenses"); - cdkLicenses.setTextContent(sourceName+"_"+ fElm.getTextContent()); - - doc.appendChild(cdkLicenses); + for (String licOfAncestors : licensesOfAncestors) { + Document document = doc.getOwnerDocument(); + Element cdkLicenses = document.createElement("field"); + cdkLicenses.setAttribute("name", "cdk.licenses_of_ancestors"); + cdkLicenses.setTextContent(sourceName+"_"+ licOfAncestors); + doc.appendChild(cdkLicenses); + } + + + List containsLicenses = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { + + @Override + public boolean acceptElement(Element element) { + String attribute = element.getAttribute("name"); + return "contains_licenses".equals(attribute); } + }).stream().map(Element::getTextContent).collect(Collectors.toList()); + + for (String licOfAncestors : containsLicenses) { + Document document = doc.getOwnerDocument(); + Element cdkLicenses = document.createElement("field"); + cdkLicenses.setAttribute("name", "cdk.contains_licenses"); + cdkLicenses.setTextContent(sourceName+"_"+ licOfAncestors); + doc.appendChild(cdkLicenses); } + + + + List licenses = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { + + @Override + public boolean acceptElement(Element element) { + String attribute = element.getAttribute("name"); + return "licenses".equals(attribute); + } + }).stream().map(Element::getTextContent).collect(Collectors.toList()); + for (String license : licenses) { + Document document = doc.getOwnerDocument(); + Element cdkLicenses = document.createElement("field"); + cdkLicenses.setAttribute("name", "cdk.licenses"); + cdkLicenses.setTextContent(sourceName+"_"+ license); + doc.appendChild(cdkLicenses); + } + } }); @@ -150,9 +192,11 @@ public ModifyFieldResult modifyField(Element field) { List deleteFields = Arrays.asList(); List addValues = Arrays.asList( + // toto musi byt pocitane pole "licenses", "licenses_of_ancestors", "contains_licenses", + "in_collections", "in_collections.direct"); @@ -174,56 +218,86 @@ public ModifyFieldResult modifyField(Element field) { @Override public void changeDocument(String root, String pid, Element doc) { + // setreseni dat cdk.licenses - // cdk dokument - Map cdkDoc = docs.get(pid); + List> comparingFields = Arrays.asList( + Pair.of("licenses", "cdk.licenses"), + Pair.of("contains_licenses", "cdk.contains_licenses"), + Pair.of("licenses_of_ancestors", "cdk.licenses_of_ancestors") + ); - List newIndexedLicenses = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { + Map cdkDoc = docs.get(pid); + for (Pair cpField : comparingFields) { - @Override - public boolean acceptElement(Element element) { - String attribute = element.getAttribute("name"); - return "licenses".equals(attribute); - } - }); - - - List newCDKLicenses = new ArrayList<>(); - newCDKLicenses = newIndexedLicenses.stream().map(Element::getTextContent).map(lic-> { - return sourceName+"_"+lic; - }).collect(Collectors.toList()); - - List indexedCDKLicenses = (List) cdkDoc.get("cdk.licenses"); - if (indexedCDKLicenses == null) { - indexedCDKLicenses = new ArrayList<>(); - } + String sourceField = cpField.getLeft(); + String specificCDKField = cpField.getRight(); + + List newIndexedField = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String attribute = element.getAttribute("name"); + return sourceField.equals(attribute); + } + }); + + Set newCDKValues = new HashSet<>(); + newCDKValues = newIndexedField.stream().map(Element::getTextContent).map(cnt-> { + return sourceName+"_"+cnt; + }).collect(Collectors.toSet()); + + + Set indexedCDKLicenses = cdkDoc.get(specificCDKField) != null ? new HashSet((List)cdkDoc.get(specificCDKField)) : new HashSet<>(); + indexedCDKLicenses.removeIf(item -> !item.startsWith(sourceName + "_")); + if (!indexedCDKLicenses.equals(newCDKValues)) { + List newList = new ArrayList( cdkDoc.get(specificCDKField) != null ? (List)cdkDoc.get(specificCDKField) : new ArrayList<>() ); + // remove everything what is prefixed + newList.removeIf(item -> item.startsWith(sourceName + "_")); + + // add new indexed values + newList.addAll(newCDKValues); - List toRemoveCDKLicenses = new ArrayList<>(indexedCDKLicenses); - toRemoveCDKLicenses.removeAll(newCDKLicenses); - - if (!newCDKLicenses.isEmpty()) { - for (Element fElm : newIndexedLicenses) { - Document document = fElm.getOwnerDocument(); - Element cdkLicenses = document.createElement("field"); - cdkLicenses.setAttribute("name", "cdk.licenses"); - cdkLicenses.setTextContent(sourceName+"_"+fElm.getTextContent()); +// // collect all libraries +// Set libraries = newList.stream().map(it-> { +// int idx = it.indexOf("_"); +// if (idx > 0) { +// return it.substring(0, idx); +// } else { +// return it; +// } +// }).collect(Collectors.toSet()); - cdkLicenses.setAttribute("update", "add-distinct"); - doc.appendChild(cdkLicenses); - } - } - - if (toRemoveCDKLicenses!=null && toRemoveCDKLicenses.size() > 0) { - for (String remove : toRemoveCDKLicenses) { + for (Element nIF : newIndexedField) { + doc.removeChild(nIF); + } + + Set tempSet = new HashSet<>(); Document document = doc.getOwnerDocument(); - Element cdkLicenses = document.createElement("field"); - cdkLicenses.setAttribute("name", "cdk.licenses"); - cdkLicenses.setTextContent(sourceName+"_"+remove); - cdkLicenses.setAttribute("update", "remove"); - doc.appendChild(cdkLicenses); + newList.stream().forEach(lic-> { + + Element cdkSpecific = document.createElement("field"); + cdkSpecific.setAttribute("name", specificCDKField); + cdkSpecific.setAttribute("update", "set"); + cdkSpecific.setTextContent(lic); + doc.appendChild(cdkSpecific); + + Pair divided = supportedLibraries.divideLibraryAndLicense(lic); + if (divided != null) { + + String rv = divided.getRight(); + if (!tempSet.contains(rv)) { + Element changedField = document.createElement("field"); + changedField.setAttribute("name", sourceField); + changedField.setAttribute("update", "set"); + + changedField.setTextContent(rv); + doc.appendChild(changedField); + + tempSet.add(rv); + } + } + }); } } - } }); } else { @@ -298,7 +372,6 @@ private Document retrieveAndCretebatch(List pids, String fl, CopyReplica Element resultElem = XMLUtils.findElement(response, (elm) -> { return elm.getNodeName().equals("result"); }); - // create batch Document batch = BatchUtils.batch(resultElem, this.compositeId, this.rootOfComposite, this.childOfComposite, this.transform, consumer); return batch; diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/registr b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/registr new file mode 100644 index 0000000000..75268691e3 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/registr @@ -0,0 +1 @@ +[{"id":15,"code":"zcm","sigla":"PNE303","name":"Západočeské muzeum v Plzni","name_en":"Museum of West Bohemia in Pilsen","alive":false,"last_state_switch":"2022-08-17T15:04:05.807Z","state_duration":36009419,"version":"5.3.3","url":"https://kramerius.zcm.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/zcm/logo","email":null,"web":"http://www.zcm.cz","street":"Kopeckého sady 2","city":"Plzeň","zip":"301 50","latitude":49.74491389,"longitude":13.37874444,"collections":0,"recommended_all":10,"recommended_public":1,"documents_all":42,"documents_public":2,"pages_all":488520,"pages_public":20345,"created_at":"2016-10-23T13:50:30.209Z","updated_at":"2023-10-08T09:40:06.139Z","last_document_at":"2016-05-05T13:26:49.279Z","last_document_before":2712,"licenses":[],"model_monograph_all":0,"model_monograph_public":0,"model_periodical_all":42,"model_periodical_public":2,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":0,"model_periodicalitem_public":0,"model_supplement_all":0,"model_supplement_public":0,"model_periodicalvolume_all":0,"model_periodicalvolume_public":0,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":10,"code":"kvkli","sigla":"LIA001","name":"Krajská vědecká knihovna v Liberci","name_en":"Regional Research Library in Liberec","alive":true,"last_state_switch":"2023-09-29T12:00:07.206Z","state_duration":769257,"version":"5.8.3","url":"https://kramerius.kvkli.cz","new_client_url":"https://kramerius.kvkli.cz","new_client_version":"2.4.4","logo":"https://api.registr.digitalniknihovna.cz/libraries/kvkli/logo","email":null,"web":"http://www.kvkli.cz","street":"Rumjancevova 1362/1","city":"Liberec","zip":"460 01","latitude":50.77184722,"longitude":15.05823889,"collections":1,"recommended_all":10,"recommended_public":5,"documents_all":4296,"documents_public":385,"pages_all":1783392,"pages_public":952083,"created_at":"2016-10-23T13:50:30.190Z","updated_at":"2023-09-29T12:00:08.452Z","last_document_at":"2023-07-18T13:58:37.828Z","last_document_before":82,"licenses":[],"model_monograph_all":4187,"model_monograph_public":316,"model_periodical_all":109,"model_periodical_public":69,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":5125,"model_article_public":4988,"model_periodicalitem_all":86609,"model_periodicalitem_public":80770,"model_supplement_all":2584,"model_supplement_public":2502,"model_periodicalvolume_all":1572,"model_periodicalvolume_public":1160,"model_monographunit_all":40,"model_monographunit_public":1,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":1584,"model_picture_public":935},{"id":26,"code":"svkos","sigla":"OSA001","name":"Moravskoslezská vědecká knihovna v Ostravě","name_en":"Moravian-Silesian Research Library in Ostrava","alive":true,"last_state_switch":"2023-09-27T13:29:00.691Z","state_duration":936724,"version":"5.7.3","url":"https://kramerius.svkos.cz","new_client_url":"https://kramerius.svkos.cz","new_client_version":"1.7.10","logo":"https://api.registr.digitalniknihovna.cz/libraries/svkos/logo","email":null,"web":"http://www.svkos.cz","street":"Prokešovo nám. 1802/9","city":"Ostrava","zip":"702 00","latitude":49.8410625,"longitude":18.29114917,"collections":20,"recommended_all":22,"recommended_public":20,"documents_all":2666,"documents_public":965,"pages_all":815989,"pages_public":432974,"created_at":"2016-10-23T13:50:30.255Z","updated_at":"2023-10-03T20:00:03.305Z","last_document_at":"2023-10-03T10:06:39.033Z","last_document_before":5,"licenses":[],"model_monograph_all":2465,"model_monograph_public":861,"model_periodical_all":172,"model_periodical_public":104,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":1,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":28,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":21358,"model_article_public":19312,"model_periodicalitem_all":68081,"model_periodicalitem_public":41516,"model_supplement_all":438,"model_supplement_public":340,"model_periodicalvolume_all":1216,"model_periodicalvolume_public":872,"model_monographunit_all":141,"model_monographunit_public":94,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":31154,"model_internalpart_public":9304,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":3487,"model_picture_public":574},{"id":32,"code":"cdk","sigla":null,"name":"Česká Digitální knihovna","name_en":"Czech Digital Library","alive":true,"last_state_switch":"2023-10-05T23:53:27.537Z","state_duration":208057,"version":"7.0.23_dev3_cdk","url":"https://api.ceskadigitalniknihovna.cz","new_client_url":"https://ceskadigitalniknihovna.cz","new_client_version":"2.5.0","logo":"https://api.registr.digitalniknihovna.cz/libraries/cdk/logo","email":null,"web":"http://ceskadigitalniknihovna.cz","street":null,"city":null,"zip":null,"latitude":null,"longitude":null,"collections":481,"recommended_all":23,"recommended_public":23,"documents_all":295393,"documents_public":65836,"pages_all":75353072,"pages_public":11829615,"created_at":"2016-10-23T13:50:30.286Z","updated_at":"2023-10-07T06:03:45.470Z","last_document_at":"2023-10-05T11:17:43.014Z","last_document_before":3,"licenses":[{"id":"dnnto","count":39290088},{"id":"onsite","count":19589786},{"id":"public","count":8538119},{"id":"dnntt","count":8267532},{"id":"covid","count":226142},{"id":"mzk_public-contract","count":171615},{"id":"public-muo","count":146186},{"id":"mzk_public-muo","count":98030},{"id":"onsite-sheetmusic","count":31132},{"id":"license","count":3885},{"id":"knav_pracovisteAV","count":3182},{"id":"knav_ustav_BU","count":1435},{"id":"_dnnto","count":442}],"model_monograph_all":251031,"model_monograph_public":32604,"model_periodical_all":5122,"model_periodical_public":1266,"model_soundrecording_all":684,"model_soundrecording_public":168,"model_map_all":13994,"model_map_public":9511,"model_graphic_all":18927,"model_graphic_public":18922,"model_sheetmusic_all":4749,"model_sheetmusic_public":2481,"model_archive_all":548,"model_archive_public":548,"model_manuscript_all":221,"model_manuscript_public":220,"model_article_all":203127,"model_article_public":99073,"model_periodicalitem_all":770926,"model_periodicalitem_public":284620,"model_supplement_all":79387,"model_supplement_public":52601,"model_periodicalvolume_all":45401,"model_periodicalvolume_public":15239,"model_monographunit_all":9484,"model_monographunit_public":2508,"model_track_all":5072,"model_track_public":535,"model_soundunit_all":1476,"model_soundunit_public":378,"model_internalpart_all":14043,"model_internalpart_public":12255,"model_convolute_all":117,"model_convolute_public":116,"model_picture_all":8401,"model_picture_public":59},{"id":31,"code":"kkvhb","sigla":"HBG001","name":"Krajská knihovna Vysočiny","name_en":"Regional Library of Highlands","alive":true,"last_state_switch":"2023-09-28T07:32:20.453Z","state_duration":871724,"version":"5.7.3","url":"https://kramerius.kkvysociny.cz","new_client_url":"https://kramerius.kkvysociny.cz","new_client_version":"2.3.5.2","logo":"https://api.registr.digitalniknihovna.cz/libraries/kkvhb/logo","email":null,"web":"https://www.kkvysociny.cz","street":"Havlíčkovo nám. 87","city":"Havlíčkův Brod","zip":"580 01","latitude":49.606275,"longitude":15.57881667,"collections":12,"recommended_all":0,"recommended_public":0,"documents_all":2044,"documents_public":306,"pages_all":542116,"pages_public":176279,"created_at":"2016-10-23T13:50:30.282Z","updated_at":"2023-09-28T20:00:59.825Z","last_document_at":"2023-09-28T13:00:59.266Z","last_document_before":10,"licenses":[],"model_monograph_all":1959,"model_monograph_public":264,"model_periodical_all":85,"model_periodical_public":42,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":20236,"model_periodicalitem_public":11841,"model_supplement_all":171,"model_supplement_public":169,"model_periodicalvolume_all":952,"model_periodicalvolume_public":529,"model_monographunit_all":2,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":54,"code":"mzkk5","sigla":null,"name":"MZK - stará K5","name_en":null,"alive":true,"last_state_switch":"2023-10-08T08:16:06.183Z","state_duration":5098,"version":"5.7.2","url":"https://kramerius.mzk.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/mzkk5/logo","email":null,"web":null,"street":null,"city":null,"zip":null,"latitude":null,"longitude":null,"collections":98,"recommended_all":25,"recommended_public":23,"documents_all":277938,"documents_public":60467,"pages_all":68076075,"pages_public":9037998,"created_at":"2023-09-04T15:35:06.650Z","updated_at":"2023-10-08T08:16:06.201Z","last_document_at":"2023-10-07T00:24:07.016Z","last_document_before":1,"licenses":[{"id":"dnnto","count":36885324},{"id":"dnntt","count":6856801},{"id":"license","count":44243},{"id":"covid","count":92}],"model_monograph_all":234184,"model_monograph_public":26546,"model_periodical_all":4222,"model_periodical_public":938,"model_soundrecording_all":683,"model_soundrecording_public":172,"model_map_all":14017,"model_map_public":10505,"model_graphic_all":18922,"model_graphic_public":18919,"model_sheetmusic_all":4988,"model_sheetmusic_public":2469,"model_archive_all":548,"model_archive_public":548,"model_manuscript_all":242,"model_manuscript_public":241,"model_article_all":81809,"model_article_public":10728,"model_periodicalitem_all":621475,"model_periodicalitem_public":242756,"model_supplement_all":73423,"model_supplement_public":48861,"model_periodicalvolume_all":34602,"model_periodicalvolume_public":9179,"model_monographunit_all":8990,"model_monographunit_public":2085,"model_track_all":5061,"model_track_public":577,"model_soundunit_all":1469,"model_soundunit_public":401,"model_internalpart_all":1299,"model_internalpart_public":1135,"model_convolute_all":132,"model_convolute_public":129,"model_picture_all":0,"model_picture_public":0},{"id":51,"code":"cro","sigla":"ABE301","name":"Český rozhlas","name_en":"Czech Radio","alive":true,"last_state_switch":"2023-09-27T13:25:12.511Z","state_duration":936952,"version":"7.0.27","url":"https://kramerius.rozhlas.cz","new_client_url":"https://kramerius.rozhlas.cz","new_client_version":"2.5.0","logo":"https://api.registr.digitalniknihovna.cz/libraries/cro/logo","email":null,"web":"https://portal.rozhlas.cz","street":null,"city":null,"zip":null,"latitude":null,"longitude":null,"collections":0,"recommended_all":5,"recommended_public":0,"documents_all":32,"documents_public":7,"pages_all":569572,"pages_public":6265,"created_at":"2022-11-02T08:34:56.576Z","updated_at":"2023-09-27T13:25:15.320Z","last_document_at":"2023-06-25T20:54:01.184Z","last_document_before":105,"licenses":[{"id":"cro_readOnly","count":166615}],"model_monograph_all":4,"model_monograph_public":1,"model_periodical_all":20,"model_periodical_public":5,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":3,"model_graphic_public":3,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":65317,"model_periodicalitem_public":147,"model_supplement_all":1964,"model_supplement_public":18,"model_periodicalvolume_all":182,"model_periodicalvolume_public":27,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":5,"model_convolute_public":1,"model_picture_all":0,"model_picture_public":0},{"id":8,"code":"mlp","sigla":"ABG001","name":"Městská knihovna v Praze","name_en":"Municipal Library of Prague","alive":true,"last_state_switch":"2023-10-07T23:12:06.350Z","state_duration":37738,"version":"5.8.4.1","url":"https://kramerius4.mlp.cz","new_client_url":"https://www.digitalniknihovna.cz/mlp","new_client_version":"2.5.0","logo":"https://api.registr.digitalniknihovna.cz/libraries/mlp/logo","email":null,"web":"http://www.mlp.cz","street":"Mariánské nám. 1/98","city":"Praha 1","zip":"115 72","latitude":50.087395,"longitude":14.41739917,"collections":32,"recommended_all":3,"recommended_public":3,"documents_all":9906,"documents_public":5546,"pages_all":1662437,"pages_public":894875,"created_at":"2016-10-23T13:50:30.180Z","updated_at":"2023-10-07T23:12:06.363Z","last_document_at":"2023-03-27T19:07:44.178Z","last_document_before":195,"licenses":[],"model_monograph_all":6371,"model_monograph_public":3377,"model_periodical_all":249,"model_periodical_public":85,"model_soundrecording_all":149,"model_soundrecording_public":1,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":3137,"model_sheetmusic_public":2083,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":1,"model_article_public":0,"model_periodicalitem_all":20716,"model_periodicalitem_public":10026,"model_supplement_all":2328,"model_supplement_public":1818,"model_periodicalvolume_all":994,"model_periodicalvolume_public":629,"model_monographunit_all":800,"model_monographunit_public":720,"model_track_all":992,"model_track_public":21,"model_soundunit_all":451,"model_soundunit_public":21,"model_internalpart_all":34301,"model_internalpart_public":15525,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":11,"code":"svkpk","sigla":"PNA001","name":"Studijní a vědecká knihovna Plzeňského kraje","name_en":"Education and Research Library of Pilsener Region, contributory organization","alive":true,"last_state_switch":"2023-10-07T00:04:06.863Z","state_duration":121018,"version":"7.0.23","url":"https://kramerius.svkpk.cz","new_client_url":"https://kramerius.svkpk.cz","new_client_version":"2.4.7","logo":"https://api.registr.digitalniknihovna.cz/libraries/svkpk/logo","email":null,"web":"https://svkpk.cz","street":"Smetanovy sady 2","city":"Plzeň","zip":"301 00","latitude":49.74460278,"longitude":13.37535556,"collections":6,"recommended_all":0,"recommended_public":0,"documents_all":140,"documents_public":71,"pages_all":573409,"pages_public":187344,"created_at":"2016-10-23T13:50:30.193Z","updated_at":"2023-10-07T00:04:06.881Z","last_document_at":"2023-09-21T08:19:31.996Z","last_document_before":17,"licenses":[],"model_monograph_all":73,"model_monograph_public":43,"model_periodical_all":64,"model_periodical_public":28,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":3,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":434173,"model_article_public":49204,"model_periodicalitem_all":79616,"model_periodicalitem_public":24274,"model_supplement_all":1020,"model_supplement_public":343,"model_periodicalvolume_all":543,"model_periodicalvolume_public":223,"model_monographunit_all":1,"model_monographunit_public":1,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":5,"model_internalpart_public":5,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":11288,"model_picture_public":998},{"id":23,"code":"uzei","sigla":"ABA009","name":"Knihovna Antonína Švehly","name_en":"Library of Antonín Švehla","alive":true,"last_state_switch":"2023-10-03T11:20:06.987Z","state_duration":426058,"version":"5.7.2","url":"https://kramerius.uzei.cz","new_client_url":"https://dk.uzei.cz","new_client_version":"2.3.5","logo":"https://api.registr.digitalniknihovna.cz/libraries/uzei/logo","email":null,"web":"http://www.knihovnasvehly.cz","street":"Slezská 7","city":"Praha 2","zip":"120 00","latitude":50.07606667,"longitude":14.43860833,"collections":0,"recommended_all":2,"recommended_public":2,"documents_all":3116,"documents_public":3032,"pages_all":637329,"pages_public":590372,"created_at":"2016-10-23T13:50:30.242Z","updated_at":"2023-10-03T11:20:07.634Z","last_document_at":"2023-09-06T11:41:03.012Z","last_document_before":32,"licenses":[],"model_monograph_all":3043,"model_monograph_public":2961,"model_periodical_all":73,"model_periodical_public":71,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":12731,"model_periodicalitem_public":11349,"model_supplement_all":1180,"model_supplement_public":939,"model_periodicalvolume_all":721,"model_periodicalvolume_public":635,"model_monographunit_all":90,"model_monographunit_public":90,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":9604,"model_internalpart_public":9538,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":18,"code":"nkp","sigla":"ABA000","name":"Národní knihovna České republiky","name_en":"National Library of the Czech Republic","alive":true,"last_state_switch":"2023-09-27T13:32:13.139Z","state_duration":936531,"version":"5.8.4.3","url":"https://www.ndk.cz","new_client_url":"https://www.ndk.cz","new_client_version":"2.4.9","logo":"https://api.registr.digitalniknihovna.cz/libraries/nkp/logo","email":"Jan.Holomek@nkp.cz","web":"http://www.nkp.cz","street":"Klementinum 190","city":"Praha 1","zip":"110 00","latitude":50.08642222,"longitude":14.415725,"collections":50,"recommended_all":31,"recommended_public":31,"documents_all":250484,"documents_public":37906,"pages_all":72919680,"pages_public":12381392,"created_at":"2016-10-23T13:50:30.222Z","updated_at":"2023-10-07T06:00:41.274Z","last_document_at":"2023-10-07T00:21:27.941Z","last_document_before":1,"licenses":[{"id":"dnnto","count":40829077},{"id":"covid","count":27168643},{"id":"dnntt","count":6803511}],"model_monograph_all":240102,"model_monograph_public":33595,"model_periodical_all":5065,"model_periodical_public":1139,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":4246,"model_map_public":2269,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":1071,"model_sheetmusic_public":903,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":961716,"model_periodicalitem_public":378805,"model_supplement_all":120635,"model_supplement_public":71486,"model_periodicalvolume_all":41173,"model_periodicalvolume_public":11470,"model_monographunit_all":13394,"model_monographunit_public":6377,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":617964,"model_internalpart_public":66647,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":5,"code":"knav","sigla":"ABA007","name":"Knihovna Akademie věd ČR","name_en":"Library of the Czech Academy of Sciences","alive":true,"last_state_switch":"2023-10-08T08:32:07.155Z","state_duration":4137,"version":"5.8.4.3","url":"https://kramerius.lib.cas.cz","new_client_url":"https://kramerius.lib.cas.cz","new_client_version":"2.4.9","logo":"https://api.registr.digitalniknihovna.cz/libraries/knav/logo","email":null,"web":"http://www.knav.cz","street":"Národní 3","city":"Praha 1","zip":"115 22","latitude":50.08142778,"longitude":14.41385556,"collections":83,"recommended_all":15,"recommended_public":15,"documents_all":8606,"documents_public":3411,"pages_all":4905419,"pages_public":2352435,"created_at":"2016-10-23T13:50:30.168Z","updated_at":"2023-10-08T08:32:07.168Z","last_document_at":"2023-10-05T11:17:43.014Z","last_document_before":3,"licenses":[{"id":"dnnto","count":1723761},{"id":"dnntt","count":137479},{"id":"knav_pracovisteAV","count":2305},{"id":"knav_ustav_BU","count":1375}],"model_monograph_all":8356,"model_monograph_public":3260,"model_periodical_all":246,"model_periodical_public":148,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":1,"model_map_public":1,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":3,"model_sheetmusic_public":2,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":128343,"model_article_public":100277,"model_periodicalitem_all":28678,"model_periodicalitem_public":16121,"model_supplement_all":487,"model_supplement_public":257,"model_periodicalvolume_all":7197,"model_periodicalvolume_public":4107,"model_monographunit_all":1032,"model_monographunit_public":601,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":100,"model_internalpart_public":33,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":1,"model_picture_public":0},{"id":3,"code":"svkhk","sigla":"HKA001","name":"Studijní a vědecká knihovna v Hradci Králové","name_en":"Research Library in Hradec Králové","alive":true,"last_state_switch":"2023-10-04T00:36:37.510Z","state_duration":378267,"version":"5.8.4.3","url":"https://kramerius.svkhk.cz","new_client_url":"https://kramerius.svkhk.cz","new_client_version":"2.4.9","logo":"https://api.registr.digitalniknihovna.cz/libraries/svkhk/logo","email":"webmaster@svkhk.cz","web":"http://www.svkhk.cz","street":"Hradecká 1250/2","city":"Hradec Králové","zip":"500 03","latitude":50.2052,"longitude":15.83149972,"collections":36,"recommended_all":22,"recommended_public":13,"documents_all":5205,"documents_public":2935,"pages_all":2093666,"pages_public":946149,"created_at":"2016-10-23T13:50:30.160Z","updated_at":"2023-10-04T00:36:37.523Z","last_document_at":"2023-08-24T13:20:15.134Z","last_document_before":45,"licenses":[{"id":"dnnto","count":997043},{"id":"dnntt","count":2936}],"model_monograph_all":4822,"model_monograph_public":2759,"model_periodical_all":382,"model_periodical_public":175,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":1,"model_sheetmusic_public":1,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":114968,"model_periodicalitem_public":49297,"model_supplement_all":3656,"model_supplement_public":336,"model_periodicalvolume_all":4408,"model_periodicalvolume_public":2308,"model_monographunit_all":206,"model_monographunit_public":153,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":582,"model_internalpart_public":574,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":1,"model_picture_public":0},{"id":14,"code":"nm","sigla":"ABA010","name":"Národní muzeum","name_en":"National Museum","alive":true,"last_state_switch":"2023-09-27T13:17:48.569Z","state_duration":937396,"version":"5.8.3","url":"https://kramerius.nm.cz","new_client_url":"https://kramerius.nm.cz","new_client_version":"2.4.9","logo":"https://api.registr.digitalniknihovna.cz/libraries/nm/logo","email":null,"web":"http://www.nm.cz","street":"Vinohradská 1","city":"Praha 1","zip":"110 00","latitude":50.07989889,"longitude":14.43171,"collections":12,"recommended_all":14,"recommended_public":11,"documents_all":1320,"documents_public":333,"pages_all":889618,"pages_public":849858,"created_at":"2016-10-23T13:50:30.205Z","updated_at":"2023-09-27T13:17:48.574Z","last_document_at":"2023-09-15T10:27:16.314Z","last_document_before":23,"licenses":[],"model_monograph_all":120,"model_monograph_public":4,"model_periodical_all":63,"model_periodical_public":60,"model_soundrecording_all":1109,"model_soundrecording_public":241,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":28,"model_sheetmusic_public":28,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":18474,"model_article_public":0,"model_periodicalitem_all":82431,"model_periodicalitem_public":80541,"model_supplement_all":2760,"model_supplement_public":2755,"model_periodicalvolume_all":969,"model_periodicalvolume_public":922,"model_monographunit_all":110,"model_monographunit_public":108,"model_track_all":2797,"model_track_public":587,"model_soundunit_all":2327,"model_soundunit_public":522,"model_internalpart_all":373,"model_internalpart_public":373,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":1,"model_picture_public":1},{"id":22,"code":"lmda","sigla":null,"name":"Lesnický a myslivecký digitální archiv","name_en":"Forestry and Hunting Digital Archive","alive":true,"last_state_switch":"2023-09-27T13:27:27.881Z","state_duration":936817,"version":"5.8.1","url":"https://lmda.silvarium.cz","new_client_url":"https://lmda.silvarium.cz","new_client_version":"2.4.9","logo":"https://api.registr.digitalniknihovna.cz/libraries/lmda/logo","email":null,"web":null,"street":null,"city":null,"zip":null,"latitude":null,"longitude":null,"collections":0,"recommended_all":0,"recommended_public":0,"documents_all":2,"documents_public":2,"pages_all":89092,"pages_public":89092,"created_at":"2016-10-23T13:50:30.238Z","updated_at":"2023-09-27T13:27:27.890Z","last_document_at":"2023-09-05T13:16:23.695Z","last_document_before":33,"licenses":[],"model_monograph_all":0,"model_monograph_public":0,"model_periodical_all":2,"model_periodical_public":2,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":1374,"model_periodicalitem_public":1374,"model_supplement_all":263,"model_supplement_public":263,"model_periodicalvolume_all":126,"model_periodicalvolume_public":126,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":12,"code":"nfa","sigla":"ABC135","name":"Národní filmový archiv","name_en":"The National Film Archive","alive":true,"last_state_switch":"2023-09-27T13:27:54.247Z","state_duration":936790,"version":"5.7.2","url":"https://library.nfa.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/nfa/logo","email":"webmaster@nfa.cz","web":"http://knihovna.nfa.cz","street":"Bartolomějská 11","city":"Praha 1","zip":"110 00","latitude":50.08307222,"longitude":14.41756111,"collections":5,"recommended_all":13,"recommended_public":13,"documents_all":311,"documents_public":72,"pages_all":457833,"pages_public":34659,"created_at":"2016-10-23T13:50:30.197Z","updated_at":"2023-09-27T13:27:54.252Z","last_document_at":"2023-02-14T14:26:43.735Z","last_document_before":236,"licenses":[],"model_monograph_all":143,"model_monograph_public":51,"model_periodical_all":168,"model_periodical_public":21,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":3622,"model_article_public":4,"model_periodicalitem_all":25356,"model_periodicalitem_public":553,"model_supplement_all":136,"model_supplement_public":1,"model_periodicalvolume_all":995,"model_periodicalvolume_public":74,"model_monographunit_all":86,"model_monographunit_public":14,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":849,"model_internalpart_public":200,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":8342,"model_picture_public":24},{"id":39,"code":"mjh","sigla":"JHE301","name":"Muzeum Jindřichohradecka","name_en":"Museum of Jindrichuv Hradec Region","alive":true,"last_state_switch":"2023-09-27T23:12:14.314Z","state_duration":901730,"version":"5.8.3","url":"https://kramerius.mjh.cz","new_client_url":"https://kramerius.mjh.cz","new_client_version":"2.5.0","logo":"https://api.registr.digitalniknihovna.cz/libraries/mjh/logo","email":null,"web":"https://www.mjh.cz/","street":null,"city":null,"zip":null,"latitude":null,"longitude":null,"collections":0,"recommended_all":0,"recommended_public":0,"documents_all":83,"documents_public":79,"pages_all":389457,"pages_public":292844,"created_at":"2018-02-16T13:36:50.286Z","updated_at":"2023-10-06T20:00:33.521Z","last_document_at":"2023-02-24T13:06:48.256Z","last_document_before":226,"licenses":[],"model_monograph_all":53,"model_monograph_public":53,"model_periodical_all":30,"model_periodical_public":26,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":658,"model_article_public":552,"model_periodicalitem_all":26354,"model_periodicalitem_public":21505,"model_supplement_all":2023,"model_supplement_public":1917,"model_periodicalvolume_all":461,"model_periodicalvolume_public":425,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":42,"code":"cuni","sigla":"ABC013","name":"Univerzita Karlova","name_en":"Charles University","alive":true,"last_state_switch":"2023-09-27T13:09:45.420Z","state_duration":937879,"version":"5.7.2","url":"https://kramerius.cuni.cz","new_client_url":"https://kramerius.cuni.cz/uk/","new_client_version":"2.3.8","logo":"https://api.registr.digitalniknihovna.cz/libraries/cuni/logo","email":null,"web":"https://cuni.cz","street":"Ovocný trh 560/5","city":"Praha 1","zip":"116 36","latitude":50.086409,"longitude":14.423851,"collections":9,"recommended_all":7,"recommended_public":7,"documents_all":33873,"documents_public":33028,"pages_all":467231,"pages_public":204266,"created_at":"2019-09-04T18:27:43.779Z","updated_at":"2023-10-06T20:00:39.092Z","last_document_at":"2023-10-06T10:14:16.863Z","last_document_before":2,"licenses":[],"model_monograph_all":1194,"model_monograph_public":349,"model_periodical_all":28,"model_periodical_public":28,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":31351,"model_map_public":31351,"model_graphic_all":1300,"model_graphic_public":1300,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":1,"model_article_public":1,"model_periodicalitem_all":6500,"model_periodicalitem_public":6468,"model_supplement_all":882,"model_supplement_public":878,"model_periodicalvolume_all":512,"model_periodicalvolume_public":483,"model_monographunit_all":41,"model_monographunit_public":10,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":35,"model_internalpart_public":35,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":9,"code":"kkkv","sigla":"KVG001","name":"Krajská knihovna Karlovy Vary","name_en":"Regional Library Karlovy Vary","alive":true,"last_state_switch":"2023-10-04T13:28:08.417Z","state_duration":331976,"version":"5.3.8","url":"https://k4.kr-karlovarsky.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/kkkv/logo","email":null,"web":"http://www.knihovnakv.cz","street":"Závodní 378/84","city":"Karlovy Vary","zip":"360 06","latitude":50.21809167,"longitude":12.82716389,"collections":0,"recommended_all":6,"recommended_public":4,"documents_all":789,"documents_public":342,"pages_all":254002,"pages_public":145787,"created_at":"2016-10-23T13:50:30.185Z","updated_at":"2023-10-04T13:28:09.013Z","last_document_at":"2020-02-26T15:11:58.359Z","last_document_before":1320,"licenses":[],"model_monograph_all":767,"model_monograph_public":327,"model_periodical_all":22,"model_periodical_public":15,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":29505,"model_periodicalitem_public":27896,"model_supplement_all":200,"model_supplement_public":67,"model_periodicalvolume_all":192,"model_periodicalvolume_public":104,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":48,"model_internalpart_public":33,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":1,"model_picture_public":0},{"id":50,"code":"eod","sigla":null,"name":"Digitální knihovna projektu EoD Open","name_en":"EoD Open project digital library","alive":true,"last_state_switch":"2023-09-27T13:29:34.967Z","state_duration":936690,"version":"7.0.29","url":"https://ds-valve.uibk.ac.at/back","new_client_url":"https://diglib.eodopen.eu","new_client_version":"2.5.0","logo":"https://api.registr.digitalniknihovna.cz/libraries/eod/logo","email":null,"web":"https://eodopen.eu","street":null,"city":null,"zip":null,"latitude":null,"longitude":null,"collections":7,"recommended_all":0,"recommended_public":0,"documents_all":4222,"documents_public":4222,"pages_all":491716,"pages_public":54517,"created_at":"2022-09-04T16:30:28.196Z","updated_at":"2023-09-27T13:29:34.975Z","last_document_at":"2023-09-18T23:37:47.702Z","last_document_before":20,"licenses":[{"id":"public","count":54517}],"model_monograph_all":4222,"model_monograph_public":4222,"model_periodical_all":0,"model_periodical_public":0,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":1,"model_map_public":1,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":6,"model_sheetmusic_public":6,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":0,"model_periodicalitem_public":0,"model_supplement_all":0,"model_supplement_public":0,"model_periodicalvolume_all":0,"model_periodicalvolume_public":0,"model_monographunit_all":36,"model_monographunit_public":36,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":6,"code":"mkct","sigla":"UOG505","name":"Městská knihovna Česká Třebová","name_en":"Municipal Library Česká Třebová","alive":false,"last_state_switch":"2023-09-07T20:33:17.365Z","state_duration":2639267,"version":"5.3.6","url":"https://k5.digiknihovna.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/mkct/logo","email":null,"web":"http://knihovna.ceska-trebova.cz","street":"Smetanova 173","city":"Česká Třebová","zip":"560 02","latitude":49.90221944,"longitude":16.443925,"collections":1,"recommended_all":7,"recommended_public":7,"documents_all":21,"documents_public":21,"pages_all":21861,"pages_public":21861,"created_at":"2016-10-23T13:50:30.172Z","updated_at":"2023-10-08T09:40:41.129Z","last_document_at":"2017-12-21T11:00:03.188Z","last_document_before":2117,"licenses":[],"model_monograph_all":5,"model_monograph_public":5,"model_periodical_all":4,"model_periodical_public":4,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":12,"model_map_public":12,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":410,"model_periodicalitem_public":410,"model_supplement_all":0,"model_supplement_public":0,"model_periodicalvolume_all":25,"model_periodicalvolume_public":25,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":29,"code":"nlk","sigla":"ABA008","name":"Národní lékařská knihovna","name_en":"National Medical Library","alive":true,"last_state_switch":"2023-10-08T01:44:08.163Z","state_duration":28616,"version":"5.5.0","url":"https://kramerius.medvik.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/nlk/logo","email":null,"web":"http://www.nlk.cz/","street":"Sokolská 54","city":"Praha 2","zip":"121 32","latitude":50.07489444,"longitude":14.42936944,"collections":0,"recommended_all":0,"recommended_public":0,"documents_all":8166,"documents_public":2235,"pages_all":2052631,"pages_public":1021206,"created_at":"2016-10-23T13:50:30.268Z","updated_at":"2023-10-08T01:44:08.174Z","last_document_at":"2023-03-06T11:25:08.000Z","last_document_before":216,"licenses":[],"model_monograph_all":6953,"model_monograph_public":1129,"model_periodical_all":322,"model_periodical_public":215,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":891,"model_graphic_public":891,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":30032,"model_article_public":14206,"model_periodicalitem_all":26476,"model_periodicalitem_public":21887,"model_supplement_all":769,"model_supplement_public":503,"model_periodicalvolume_all":2962,"model_periodicalvolume_public":2390,"model_monographunit_all":30946,"model_monographunit_public":157,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":9024,"model_internalpart_public":1134,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":37,"code":"mvchk","sigla":"HKE302","name":"Muzeum východních Čech v Hradci Králové","name_en":"The Museum of Eastern Bohemia in Hradec Králové","alive":true,"last_state_switch":"2023-09-27T13:30:08.389Z","state_duration":936656,"version":"4.6.2","url":"http://k4.muzeumhk.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/mvchk/logo","email":null,"web":"http://www.muzeumhk.cz/","street":"Eliščino nábřeží 465","city":"Hradec Králové","zip":"500 01","latitude":null,"longitude":null,"collections":null,"recommended_all":null,"recommended_public":null,"documents_all":null,"documents_public":null,"pages_all":null,"pages_public":null,"created_at":"2017-12-21T12:43:11.232Z","updated_at":"2023-09-27T13:30:08.393Z","last_document_at":null,"last_document_before":null,"licenses":[],"model_monograph_all":0,"model_monograph_public":0,"model_periodical_all":0,"model_periodical_public":0,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":0,"model_periodicalitem_public":0,"model_supplement_all":0,"model_supplement_public":0,"model_periodicalvolume_all":0,"model_periodicalvolume_public":0,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":13,"code":"zmp","sigla":"ABE323","name":"Židovské muzeum v Praze","name_en":"Jewish Museum in Prague","alive":true,"last_state_switch":"2023-09-27T13:30:08.485Z","state_duration":936656,"version":"5.8.4.3","url":"https://kramerius.jewishmuseum.cz","new_client_url":"https://kramerius.jewishmuseum.cz","new_client_version":"2.4.9","logo":"https://api.registr.digitalniknihovna.cz/libraries/zmp/logo","email":null,"web":"https://www.jewishmuseum.cz","street":"U Staré školy 1","city":"Praha 1","zip":"110 01","latitude":50.09034722,"longitude":14.42111389,"collections":0,"recommended_all":2,"recommended_public":2,"documents_all":186,"documents_public":163,"pages_all":188154,"pages_public":91756,"created_at":"2016-10-23T13:50:30.201Z","updated_at":"2023-09-27T13:30:08.489Z","last_document_at":"2023-03-27T14:30:15.296Z","last_document_before":195,"licenses":[],"model_monograph_all":158,"model_monograph_public":148,"model_periodical_all":28,"model_periodical_public":15,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":12709,"model_periodicalitem_public":5718,"model_supplement_all":1264,"model_supplement_public":359,"model_periodicalvolume_all":505,"model_periodicalvolume_public":232,"model_monographunit_all":8,"model_monographunit_public":8,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":60,"model_internalpart_public":2,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":1,"code":"mzk","sigla":"BOA001","name":"Moravská zemská knihovna v Brně","name_en":"Moravian Library in Brno","alive":true,"last_state_switch":"2023-10-06T08:20:41.439Z","state_duration":177623,"version":"7.0.29","url":"https://api.kramerius.mzk.cz","new_client_url":"https://www.digitalniknihovna.cz/mzk/","new_client_version":"2.5.0","logo":"https://api.registr.digitalniknihovna.cz/libraries/mzk/logo","email":null,"web":"http://www.mzk.cz","street":"Kounicova 65a","city":"Brno","zip":"601 87","latitude":49.20865,"longitude":16.59402778,"collections":505,"recommended_all":25,"recommended_public":23,"documents_all":277851,"documents_public":56538,"pages_all":68114634,"pages_public":8413698,"created_at":"2016-10-23T13:50:30.150Z","updated_at":"2023-10-08T06:01:09.826Z","last_document_at":"2023-10-06T08:15:21.379Z","last_document_before":2,"licenses":[{"id":"dnnto","count":38000763},{"id":"onsite","count":14854347},{"id":"public","count":7953683},{"id":"dnntt","count":6876272},{"id":"mzk_public-contract","count":360735},{"id":"covid","count":277819},{"id":"mzk_public-muo","count":196776},{"id":"onsite-sheetmusic","count":20387},{"id":"license","count":2731},{"id":"public-muo","count":241}],"model_monograph_all":233953,"model_monograph_public":23933,"model_periodical_all":4255,"model_periodical_public":844,"model_soundrecording_all":683,"model_soundrecording_public":168,"model_map_all":14011,"model_map_public":9538,"model_graphic_all":18907,"model_graphic_public":18904,"model_sheetmusic_all":5143,"model_sheetmusic_public":2256,"model_archive_all":548,"model_archive_public":548,"model_manuscript_all":221,"model_manuscript_public":220,"model_article_all":81813,"model_article_public":11266,"model_periodicalitem_all":615814,"model_periodicalitem_public":238832,"model_supplement_all":73254,"model_supplement_public":52295,"model_periodicalvolume_all":34443,"model_periodicalvolume_public":9158,"model_monographunit_all":8832,"model_monographunit_public":1755,"model_track_all":5059,"model_track_public":561,"model_soundunit_all":1469,"model_soundunit_public":391,"model_internalpart_all":1316,"model_internalpart_public":1135,"model_convolute_all":130,"model_convolute_public":127,"model_picture_all":56,"model_picture_public":56},{"id":43,"code":"kkp","sigla":"PAG001","name":"Krajská knihovna v Pardubicích","name_en":"Regional Library in Pardubice","alive":true,"last_state_switch":"2023-09-27T13:31:09.971Z","state_duration":936595,"version":"5.8.3","url":"https://kramerius.knihovna-pardubice.cz","new_client_url":"https://kramerius.knihovna-pardubice.cz","new_client_version":"2.4.9","logo":"https://api.registr.digitalniknihovna.cz/libraries/kkp/logo","email":null,"web":"https://kkpce.cz","street":"Pernštýnské nám. 77","city":"Pardubice","zip":"53094","latitude":50.0381814,"longitude":15.7781661,"collections":0,"recommended_all":0,"recommended_public":0,"documents_all":129,"documents_public":67,"pages_all":148797,"pages_public":19779,"created_at":"2019-10-06T23:10:25.737Z","updated_at":"2023-10-04T20:02:39.760Z","last_document_at":"2023-10-04T19:11:29.939Z","last_document_before":4,"licenses":[],"model_monograph_all":101,"model_monograph_public":62,"model_periodical_all":28,"model_periodical_public":5,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":12610,"model_periodicalitem_public":997,"model_supplement_all":614,"model_supplement_public":54,"model_periodicalvolume_all":257,"model_periodicalvolume_public":22,"model_monographunit_all":2,"model_monographunit_public":2,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":1626,"model_internalpart_public":1141,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":1,"model_picture_public":1},{"id":46,"code":"hmt","sigla":"TAE301","name":"Husitské muzeum v Táboře","name_en":"Hussite Museum in Tabor","alive":true,"last_state_switch":"2023-09-27T13:30:38.971Z","state_duration":936626,"version":"5.4.2","url":"http://kramerius.husitskemuzeum.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/hmt/logo","email":null,"web":"http://www.husitskemuzeum.cz/knihovna/","street":"náměstí Mikuláše z Husi 44","city":"Tábor","zip":"39001","latitude":49.4139567,"longitude":14.6556442,"collections":0,"recommended_all":0,"recommended_public":0,"documents_all":5,"documents_public":5,"pages_all":27814,"pages_public":27814,"created_at":"2020-09-10T07:08:02.749Z","updated_at":"2023-09-27T13:30:38.978Z","last_document_at":"2019-02-18T03:35:14.188Z","last_document_before":1693,"licenses":[],"model_monograph_all":0,"model_monograph_public":0,"model_periodical_all":5,"model_periodical_public":5,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":4022,"model_periodicalitem_public":4022,"model_supplement_all":17,"model_supplement_public":17,"model_periodicalvolume_all":95,"model_periodicalvolume_public":95,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":45,"code":"snk","sigla":null,"name":"Katalóg digitálnej knižnice","name_en":"Slovak Digital Library Catalogue","alive":false,"last_state_switch":"2023-06-12T08:24:11.656Z","state_duration":10199813,"version":"7.0.23","url":"https://onk.snk.sk","new_client_url":"https://onk.snk.sk/","new_client_version":"-","logo":"https://api.registr.digitalniknihovna.cz/libraries/snk/logo","email":null,"web":"https://www.snk.sk/sk/","street":"J. C. Hronského 1","city":"Martin","zip":"036 01","latitude":49.0687,"longitude":18.9335283,"collections":12,"recommended_all":0,"recommended_public":0,"documents_all":235267,"documents_public":15609,"pages_all":54376799,"pages_public":3731744,"created_at":"2020-06-10T00:08:50.930Z","updated_at":"2023-10-08T09:40:42.932Z","last_document_at":"2023-06-10T20:35:57.299Z","last_document_before":120,"licenses":[{"id":"only_in_library","count":40210480},{"id":"paying_users","count":10464342},{"id":"public","count":3761541},{"id":"not_accessible","count":1575}],"model_monograph_all":230960,"model_monograph_public":13909,"model_periodical_all":2732,"model_periodical_public":125,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":473,"model_map_public":4,"model_graphic_all":1575,"model_graphic_public":1575,"model_sheetmusic_all":196,"model_sheetmusic_public":69,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":4212,"model_manuscript_public":3952,"model_article_all":704543,"model_article_public":1334,"model_periodicalitem_all":375136,"model_periodicalitem_public":8927,"model_supplement_all":27894,"model_supplement_public":372,"model_periodicalvolume_all":29700,"model_periodicalvolume_public":704,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":125101,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":1256925,"model_picture_public":280},{"id":44,"code":"npmk","sigla":"ABA012","name":"Národní pedagogické muzeum J. A. Komenského","name_en":"National Pedagogical museum and Library of J. A. Comenius","alive":true,"last_state_switch":"2023-10-05T14:04:40.438Z","state_duration":243384,"version":"5.5.0","url":"https://kramerius.npmk.cz","new_client_url":"https://kramerius.npmk.cz","new_client_version":"1.7.10","logo":"https://api.registr.digitalniknihovna.cz/libraries/npmk/logo","email":null,"web":"https://www.npmk.cz","street":"Jeruzalémská 12","city":"Praha 1","zip":"110 00","latitude":null,"longitude":null,"collections":4,"recommended_all":0,"recommended_public":0,"documents_all":826,"documents_public":529,"pages_all":109753,"pages_public":70577,"created_at":"2020-01-22T14:09:40.750Z","updated_at":"2023-10-05T20:02:27.564Z","last_document_at":"2023-09-22T10:08:56.179Z","last_document_before":16,"licenses":[],"model_monograph_all":826,"model_monograph_public":529,"model_periodical_all":0,"model_periodical_public":0,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":0,"model_periodicalitem_public":0,"model_supplement_all":0,"model_supplement_public":0,"model_periodicalvolume_all":0,"model_periodicalvolume_public":0,"model_monographunit_all":2,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":27,"code":"vugtk","sigla":null,"name":"Výzkumný ústav geodetický, topografický a kartografický","name_en":"Research Institute of Geodesy, Topography and Cartography","alive":false,"last_state_switch":"2023-09-08T11:22:39.633Z","state_duration":2585905,"version":"5.4.2","url":"https://kramerius.vugtk.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/vugtk/logo","email":"webmaster@vugtk.cz","web":"http://www.vugtk.cz","street":"Ústecká 98","city":"Zdiby","zip":"250 66","latitude":50.17133889,"longitude":14.45075556,"collections":0,"recommended_all":0,"recommended_public":0,"documents_all":15,"documents_public":3,"pages_all":125469,"pages_public":71578,"created_at":"2016-10-23T13:50:30.259Z","updated_at":"2023-10-08T09:37:13.837Z","last_document_at":"2019-06-06T15:29:19.150Z","last_document_before":1585,"licenses":[],"model_monograph_all":3,"model_monograph_public":0,"model_periodical_all":12,"model_periodical_public":3,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":1850,"model_periodicalitem_public":570,"model_supplement_all":104,"model_supplement_public":55,"model_periodicalvolume_all":256,"model_periodicalvolume_public":68,"model_monographunit_all":3,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":2,"code":"vkol","sigla":"OLA001","name":"Vědecká knihovna v Olomouci","name_en":"Research Library in Olomouc","alive":true,"last_state_switch":"2023-10-05T09:00:50.878Z","state_duration":261614,"version":"5.3.3","url":"https://kramerius.kr-olomoucky.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/vkol/logo","email":"kramerius4@gmail.com","web":"http://www.vkol.cz","street":"Bezručova 659/2","city":"Olomouc","zip":"779 11","latitude":49.59351667,"longitude":17.24674444,"collections":9,"recommended_all":26,"recommended_public":24,"documents_all":2349,"documents_public":2225,"pages_all":3648264,"pages_public":3174335,"created_at":"2016-10-23T13:50:30.156Z","updated_at":"2023-10-05T09:00:50.899Z","last_document_at":"2023-10-03T13:18:52.789Z","last_document_before":5,"licenses":[],"model_monograph_all":1417,"model_monograph_public":1344,"model_periodical_all":219,"model_periodical_public":173,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":586,"model_map_public":582,"model_graphic_all":15,"model_graphic_public":15,"model_sheetmusic_all":1,"model_sheetmusic_public":1,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":111,"model_manuscript_public":110,"model_article_all":52431,"model_article_public":52431,"model_periodicalitem_all":493128,"model_periodicalitem_public":433499,"model_supplement_all":3934,"model_supplement_public":1778,"model_periodicalvolume_all":3446,"model_periodicalvolume_public":2960,"model_monographunit_all":34,"model_monographunit_public":32,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":3483,"model_internalpart_public":3464,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":35,"code":"d","sigla":null,"name":"DiFMOE","name_en":"DiFMOE","alive":true,"last_state_switch":"2023-10-01T22:38:52.639Z","state_duration":558132,"version":"5.5.0","url":"https://kramerius.difmoe.eu","new_client_url":"https://www.difmoe.eu","new_client_version":"2.4.8","logo":"https://api.registr.digitalniknihovna.cz/libraries/d/logo","email":null,"web":"www.difmoe.eu","street":null,"city":null,"zip":null,"latitude":null,"longitude":null,"collections":9,"recommended_all":15,"recommended_public":15,"documents_all":6770,"documents_public":6737,"pages_all":2360892,"pages_public":2355508,"created_at":"2017-05-18T11:01:25.856Z","updated_at":"2023-10-03T20:05:44.949Z","last_document_at":"2023-10-03T06:24:01.956Z","last_document_before":5,"licenses":[],"model_monograph_all":1126,"model_monograph_public":1095,"model_periodical_all":258,"model_periodical_public":257,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":47,"model_map_public":47,"model_graphic_all":5157,"model_graphic_public":5156,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":179,"model_archive_public":179,"model_manuscript_all":3,"model_manuscript_public":3,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":286699,"model_periodicalitem_public":286698,"model_supplement_all":0,"model_supplement_public":0,"model_periodicalvolume_all":2774,"model_periodicalvolume_public":2757,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":16,"code":"cbvk","sigla":"CBA001","name":"Jihočeská vědecká knihovna v Českých Budějovicích","name_en":"Research Library of South Bohemia in České Budějovice","alive":true,"last_state_switch":"2023-10-06T06:57:11.504Z","state_duration":182633,"version":"7.0.30-dev","url":"https://kramerius.cbvk.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/cbvk/logo","email":null,"web":"https://www.cbvk.cz","street":"Na Sadech 26","city":"České Budějovice","zip":"370 59","latitude":48.97803333,"longitude":14.47613056,"collections":0,"recommended_all":33,"recommended_public":26,"documents_all":2861,"documents_public":2548,"pages_all":3011028,"pages_public":1707626,"created_at":"2016-10-23T13:50:30.213Z","updated_at":"2023-10-06T20:03:42.929Z","last_document_at":"2023-09-20T13:05:42.261Z","last_document_before":18,"licenses":[],"model_monograph_all":1066,"model_monograph_public":915,"model_periodical_all":326,"model_periodical_public":237,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":1003,"model_map_public":930,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":466,"model_sheetmusic_public":466,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":6074,"model_article_public":922,"model_periodicalitem_all":210190,"model_periodicalitem_public":153680,"model_supplement_all":9678,"model_supplement_public":2331,"model_periodicalvolume_all":5263,"model_periodicalvolume_public":3672,"model_monographunit_all":265,"model_monographunit_public":258,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":2659,"model_internalpart_public":1856,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":27,"model_picture_public":17},{"id":41,"code":"upm","sigla":"ABE310","name":"Uměleckoprůmyslové museum v Praze","name_en":"Museum of Decorative Arts in Prague","alive":true,"last_state_switch":"2023-10-03T00:42:01.484Z","state_duration":464343,"version":"5.8.4.3","url":"https://kramerius.upm.cz","new_client_url":"https://kramerius.upm.cz","new_client_version":"2.4.9","logo":"https://api.registr.digitalniknihovna.cz/libraries/upm/logo","email":"webmaster@upm.cz","web":"https://www.upm.cz/knihovna/","street":"17. listopadu 2 ","city":"Praha 1 – Staré Město","zip":"110 00","latitude":50.0899289,"longitude":14.4163139,"collections":0,"recommended_all":11,"recommended_public":1,"documents_all":18,"documents_public":1,"pages_all":89672,"pages_public":11652,"created_at":"2019-04-12T18:41:22.807Z","updated_at":"2023-10-03T00:42:02.117Z","last_document_at":"2022-02-11T13:59:36.438Z","last_document_before":604,"licenses":[],"model_monograph_all":0,"model_monograph_public":0,"model_periodical_all":18,"model_periodical_public":1,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":3395,"model_periodicalitem_public":642,"model_supplement_all":123,"model_supplement_public":0,"model_periodicalvolume_all":232,"model_periodicalvolume_public":34,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":52,"code":"cvti","sigla":null,"name":"Centrum vedecko-technických informácií SR","name_en":"Slovak centre of scientific and technical information","alive":true,"last_state_switch":"2023-09-27T13:31:41.812Z","state_duration":936563,"version":"7.0.23","url":"https://ekniznice.cvtisr.sk","new_client_url":"https://ekniznice.cvtisr.sk/","new_client_version":"2.4.9","logo":"https://api.registr.digitalniknihovna.cz/libraries/cvti/logo","email":null,"web":"https://www.cvtisr.sk/","street":"Lamačská cesta 8/A","city":"Bratislava - Staré Mesto","zip":"840 05","latitude":48.1701369,"longitude":17.0748442,"collections":14,"recommended_all":null,"recommended_public":null,"documents_all":98,"documents_public":98,"pages_all":188640,"pages_public":188640,"created_at":"2023-01-25T17:27:56.057Z","updated_at":"2023-10-06T20:03:40.438Z","last_document_at":"2023-10-06T07:44:17.338Z","last_document_before":2,"licenses":[],"model_monograph_all":79,"model_monograph_public":79,"model_periodical_all":19,"model_periodical_public":19,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":1727,"model_periodicalitem_public":1727,"model_supplement_all":0,"model_supplement_public":0,"model_periodicalvolume_all":350,"model_periodicalvolume_public":350,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":19,"code":"cuni_fsv","sigla":"ABD103","name":"Univerzita Karlova - Fakulta sociálních věd","name_en":"Charles University - Faculty of Social Sciences","alive":true,"last_state_switch":"2023-09-27T13:00:32.225Z","state_duration":938432,"version":"5.7.2","url":"https://kramerius.fsv.cuni.cz","new_client_url":"https://kramerius.cuni.cz/fsv","new_client_version":"2.3.8","logo":"https://api.registr.digitalniknihovna.cz/libraries/cuni_fsv/logo","email":null,"web":"https://fsv.cuni.cz/","street":"Smetanovo nábř. 995/6","city":"Praha 1","zip":"110 01","latitude":50.08233889,"longitude":14.41330556,"collections":1,"recommended_all":0,"recommended_public":0,"documents_all":102,"documents_public":66,"pages_all":576358,"pages_public":129906,"created_at":"2016-10-23T13:50:30.226Z","updated_at":"2023-10-08T06:03:38.159Z","last_document_at":"2022-01-13T13:08:23.887Z","last_document_before":633,"licenses":[],"model_monograph_all":14,"model_monograph_public":9,"model_periodical_all":85,"model_periodical_public":54,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":3,"model_manuscript_public":3,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":44846,"model_periodicalitem_public":15616,"model_supplement_all":2938,"model_supplement_public":1058,"model_periodicalvolume_all":486,"model_periodicalvolume_public":188,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":1,"model_internalpart_public":1,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":40,"code":"vsup","sigla":"ABD024","name":"Vysoká škola uměleckoprůmyslová v Praze","name_en":"Academy of Arts, Architecture and Design in Prague","alive":true,"last_state_switch":"2023-10-04T14:17:21.696Z","state_duration":329023,"version":"5.8.3","url":"https://kramerius.vsup.cz","new_client_url":"https://kramerius.vsup.cz","new_client_version":"2.4.2","logo":"https://api.registr.digitalniknihovna.cz/libraries/vsup/logo","email":null,"web":"https://www.umprum.cz/","street":"Náměstí Jana Palacha 80","city":"Praha 1","zip":"116 93","latitude":null,"longitude":null,"collections":0,"recommended_all":0,"recommended_public":0,"documents_all":135,"documents_public":62,"pages_all":22729,"pages_public":8360,"created_at":"2019-03-26T13:39:54.668Z","updated_at":"2023-10-04T14:17:22.879Z","last_document_at":"2023-03-13T06:45:06.656Z","last_document_before":209,"licenses":[],"model_monograph_all":133,"model_monograph_public":62,"model_periodical_all":2,"model_periodical_public":0,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":480,"model_periodicalitem_public":0,"model_supplement_all":0,"model_supplement_public":0,"model_periodicalvolume_all":28,"model_periodicalvolume_public":0,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":7,"code":"dsmo","sigla":null,"name":"Digitální studovna Ministerstva obrany ČR","name_en":"Digital Reading Room of the Ministry of Defence of the Czech Republic","alive":true,"last_state_switch":"2023-10-07T14:13:17.224Z","state_duration":70067,"version":"5.8.1","url":"https://digitalnistudovna.army.cz","new_client_url":"https://digitalnistudovna.army.cz","new_client_version":"2.4.9","logo":"https://api.registr.digitalniknihovna.cz/libraries/dsmo/logo","email":null,"web":null,"street":null,"city":null,"zip":null,"latitude":null,"longitude":null,"collections":12,"recommended_all":0,"recommended_public":0,"documents_all":3557,"documents_public":3542,"pages_all":2317563,"pages_public":2311390,"created_at":"2016-10-23T13:50:30.176Z","updated_at":"2023-10-07T14:13:17.240Z","last_document_at":"2023-09-27T13:49:27.448Z","last_document_before":11,"licenses":[],"model_monograph_all":3344,"model_monograph_public":3330,"model_periodical_all":204,"model_periodical_public":203,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":1,"model_map_public":1,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":8,"model_sheetmusic_public":8,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":273370,"model_article_public":273370,"model_periodicalitem_all":73057,"model_periodicalitem_public":73033,"model_supplement_all":7831,"model_supplement_public":7831,"model_periodicalvolume_all":1838,"model_periodicalvolume_public":1833,"model_monographunit_all":1121,"model_monographunit_public":1121,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":941,"model_internalpart_public":941,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":116105,"model_picture_public":116018},{"id":28,"code":"vse","sigla":"ABA006","name":"Vysoká škola ekonomická v Praze","name_en":"University of Economics, Prague","alive":true,"last_state_switch":"2023-09-27T13:34:19.071Z","state_duration":936406,"version":"5.8.3","url":"https://kramerius.vse.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/vse/logo","email":"knihovna@vse.cz","web":"http://library.vse.cz","street":"nám. W. Churchilla 4","city":"Praha 3","zip":"130 67","latitude":50.08441667,"longitude":14.44115,"collections":5,"recommended_all":22,"recommended_public":22,"documents_all":1379,"documents_public":1375,"pages_all":211956,"pages_public":211093,"created_at":"2016-10-23T13:50:30.264Z","updated_at":"2023-09-27T13:34:19.076Z","last_document_at":"2022-11-17T08:26:59.444Z","last_document_before":325,"licenses":[],"model_monograph_all":1377,"model_monograph_public":1373,"model_periodical_all":2,"model_periodical_public":2,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":50,"model_periodicalitem_public":50,"model_supplement_all":32,"model_supplement_public":30,"model_periodicalvolume_all":14,"model_periodicalvolume_public":14,"model_monographunit_all":50,"model_monographunit_public":50,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":9076,"model_internalpart_public":9025,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":9,"model_picture_public":9},{"id":36,"code":"cuni_lf1","sigla":"ABD134","name":"Univerzita Karlova - 1. lékařská fakulta","name_en":"Charles University - First Faculty of Medicine","alive":true,"last_state_switch":"2023-10-03T11:10:12.639Z","state_duration":426652,"version":"5.7.2","url":"https://kramerius.lf1.cuni.cz","new_client_url":"https://kramerius.cuni.cz/lf1","new_client_version":"2.3.8","logo":"https://api.registr.digitalniknihovna.cz/libraries/cuni_lf1/logo","email":"knihovna.kramerius@lf1.cuni.cz","web":"https://www.lf1.cuni.cz/","street":null,"city":null,"zip":null,"latitude":null,"longitude":null,"collections":0,"recommended_all":0,"recommended_public":0,"documents_all":35,"documents_public":29,"pages_all":75584,"pages_public":18819,"created_at":"2017-06-01T19:35:43.677Z","updated_at":"2023-10-03T11:10:12.657Z","last_document_at":"2023-09-14T07:33:52.954Z","last_document_before":24,"licenses":[],"model_monograph_all":31,"model_monograph_public":26,"model_periodical_all":4,"model_periodical_public":3,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":440,"model_periodicalitem_public":26,"model_supplement_all":2,"model_supplement_public":0,"model_periodicalvolume_all":149,"model_periodicalvolume_public":26,"model_monographunit_all":5,"model_monographunit_public":2,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":47,"code":"nulk","sigla":"HOE801","name":"Národní ústav lidové kultury","name_en":"National Institute of Folk Culture","alive":true,"last_state_switch":"2023-09-27T13:28:24.033Z","state_duration":936761,"version":"5.8.4.3","url":"https://kramerius.nulk.cz","new_client_url":"https://kramerius.nulk.cz/","new_client_version":"2.3.9","logo":"https://api.registr.digitalniknihovna.cz/libraries/nulk/logo","email":"webmaster@nulk.cz","web":"http://www.nulk.cz","street":"Zámek 672","city":"Strážnice","zip":"696 62","latitude":48.9078592,"longitude":17.3131147,"collections":0,"recommended_all":0,"recommended_public":0,"documents_all":58,"documents_public":36,"pages_all":40711,"pages_public":23460,"created_at":"2021-01-20T10:14:50.806Z","updated_at":"2023-09-27T13:28:24.051Z","last_document_at":"2023-04-24T12:05:21.907Z","last_document_before":167,"licenses":[],"model_monograph_all":53,"model_monograph_public":33,"model_periodical_all":5,"model_periodical_public":3,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":248,"model_periodicalitem_public":164,"model_supplement_all":4,"model_supplement_public":4,"model_periodicalvolume_all":142,"model_periodicalvolume_public":106,"model_monographunit_all":13,"model_monographunit_public":9,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":810,"model_internalpart_public":631,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":17,"code":"kfbz","sigla":"ZLG001","name":"Krajská knihovna Františka Bartoše ve Zlíně","name_en":"František Bartoš Regional Library in Zlín","alive":true,"last_state_switch":"2023-10-04T11:50:13.467Z","state_duration":337851,"version":"7.0.27","url":"https://kramerius.kfbz.cz","new_client_url":"https://kramerius.kfbz.cz","new_client_version":"2.4.9","logo":"https://api.registr.digitalniknihovna.cz/libraries/kfbz/logo","email":null,"web":"https://www.kfbz.cz","street":"Vavrečkova 7040","city":"Zlín","zip":"760 01","latitude":49.22487111,"longitude":17.65892806,"collections":0,"recommended_all":0,"recommended_public":0,"documents_all":2754,"documents_public":113,"pages_all":392263,"pages_public":54397,"created_at":"2016-10-23T13:50:30.218Z","updated_at":"2023-10-04T11:50:13.472Z","last_document_at":"2023-05-26T12:00:35.893Z","last_document_before":135,"licenses":[{"id":"dnnto","count":5684},{"id":"covid","count":69}],"model_monograph_all":665,"model_monograph_public":108,"model_periodical_all":78,"model_periodical_public":5,"model_soundrecording_all":2011,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":1474,"model_article_public":611,"model_periodicalitem_all":19070,"model_periodicalitem_public":2802,"model_supplement_all":4007,"model_supplement_public":371,"model_periodicalvolume_all":667,"model_periodicalvolume_public":79,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":53643,"model_track_public":0,"model_soundunit_all":2011,"model_soundunit_public":0,"model_internalpart_all":11317,"model_internalpart_public":1666,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":5067,"model_picture_public":375},{"id":48,"code":"hiu","sigla":"ABB036","name":"Historický ústav AV ČR","name_en":"Institute of History of the AS CR","alive":false,"last_state_switch":"2023-06-12T14:52:27.856Z","state_duration":10176517,"version":"5.7.3","url":"https://kramerius.hiu.cas.cz","new_client_url":"https://kramerius.hiu.cas.cz","new_client_version":"-","logo":"https://api.registr.digitalniknihovna.cz/libraries/hiu/logo","email":null,"web":"https://www.hiu.cas.cz","street":"Prosecká 809/76","city":"Praha 9","zip":"190 00","latitude":50.1204644,"longitude":14.5052814,"collections":0,"recommended_all":0,"recommended_public":0,"documents_all":1,"documents_public":1,"pages_all":80,"pages_public":80,"created_at":"2021-12-15T14:58:05.299Z","updated_at":"2023-10-08T09:37:30.944Z","last_document_at":"2021-12-15T10:08:32.084Z","last_document_before":662,"licenses":[],"model_monograph_all":0,"model_monograph_public":0,"model_periodical_all":1,"model_periodical_public":1,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":1,"model_periodicalitem_public":1,"model_supplement_all":1,"model_supplement_public":1,"model_periodicalvolume_all":1,"model_periodicalvolume_public":1,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":0,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":4,"code":"svkul","sigla":"ULG001","name":"Knihovna Ústeckého kraje","name_en":"Ústí Regional Library","alive":true,"last_state_switch":"2023-09-28T15:06:43.593Z","state_duration":844461,"version":"5.8.3","url":"https://kramerius.svkul.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/svkul/logo","email":"maly@svkul.cz","web":"http://www.svkul.cz","street":"W. Churchilla 3","city":"Ústí nad Labem","zip":"400 01","latitude":50.6631697,"longitude":14.0354472,"collections":2,"recommended_all":18,"recommended_public":13,"documents_all":1054,"documents_public":167,"pages_all":548271,"pages_public":120408,"created_at":"2016-10-23T13:50:30.164Z","updated_at":"2023-10-03T20:07:02.634Z","last_document_at":"2023-10-03T12:07:13.054Z","last_document_before":5,"licenses":[],"model_monograph_all":918,"model_monograph_public":155,"model_periodical_all":38,"model_periodical_public":11,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":98,"model_map_public":1,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":43275,"model_periodicalitem_public":11718,"model_supplement_all":11,"model_supplement_public":0,"model_periodicalvolume_all":414,"model_periodicalvolume_public":135,"model_monographunit_all":39,"model_monographunit_public":28,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":9,"model_internalpart_public":9,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":53,"code":"dikda","sigla":null,"name":"Katalóg digitálnej knižnice","name_en":"Slovak Digital Library Catalogue","alive":true,"last_state_switch":"2023-10-08T03:05:44.217Z","state_duration":23720,"version":"7.0.27","url":"https://dikda.snk.sk","new_client_url":"https://dikda.snk.sk","new_client_version":"2.4.9","logo":"https://api.registr.digitalniknihovna.cz/libraries/dikda/logo","email":null,"web":"https://www.snk.sk/sk/","street":"J. C. Hronského 1","city":"Martin","zip":"036 01","latitude":49.0687,"longitude":18.9335283,"collections":29,"recommended_all":null,"recommended_public":null,"documents_all":238478,"documents_public":17952,"pages_all":57498734,"pages_public":3954187,"created_at":"2023-01-27T07:19:37.707Z","updated_at":"2023-10-08T06:04:32.681Z","last_document_at":"2023-10-06T11:52:40.305Z","last_document_before":2,"licenses":[{"id":"only_in_library","count":43468757},{"id":"paying_users","count":10216471},{"id":"public","count":3814748},{"id":"snk_limited_access","count":216377},{"id":"snk_only_in_archive","count":57067},{"id":"limited_access","count":158},{"id":"not_accessible","count":3096}],"model_monograph_all":230615,"model_monograph_public":13909,"model_periodical_all":3214,"model_periodical_public":125,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":312,"model_map_public":312,"model_graphic_all":3096,"model_graphic_public":2608,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":1241,"model_manuscript_public":998,"model_article_all":704605,"model_article_public":1335,"model_periodicalitem_all":467499,"model_periodicalitem_public":8927,"model_supplement_all":37233,"model_supplement_public":372,"model_periodicalvolume_all":36509,"model_periodicalvolume_public":704,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":125101,"model_internalpart_public":0,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":1256969,"model_picture_public":280},{"id":20,"code":"ntk","sigla":"ABA013","name":"Národní technická knihovna","name_en":"National Library of Technology","alive":true,"last_state_switch":"2023-10-04T19:45:38.616Z","state_duration":309326,"version":"5.8.3","url":"https://kramerius.techlib.cz","new_client_url":"https://kramerius.techlib.cz/kramerius-web-client/","new_client_version":"2.0.5","logo":"https://api.registr.digitalniknihovna.cz/libraries/ntk/logo","email":null,"web":"https://www.techlib.cz","street":"Technická 6/2710","city":"Praha 6","zip":"160 80","latitude":50.10391111,"longitude":14.39065556,"collections":17,"recommended_all":15,"recommended_public":15,"documents_all":2200,"documents_public":655,"pages_all":562339,"pages_public":229768,"created_at":"2016-10-23T13:50:30.230Z","updated_at":"2023-10-06T20:04:51.490Z","last_document_at":"2023-10-06T15:54:33.895Z","last_document_before":2,"licenses":[],"model_monograph_all":2167,"model_monograph_public":625,"model_periodical_all":30,"model_periodical_public":27,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":3,"model_map_public":3,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":1,"model_article_public":1,"model_periodicalitem_all":6675,"model_periodicalitem_public":5766,"model_supplement_all":33,"model_supplement_public":33,"model_periodicalvolume_all":516,"model_periodicalvolume_public":432,"model_monographunit_all":58,"model_monographunit_public":44,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":773,"model_internalpart_public":773,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":34,"code":"nacr","sigla":"ABE343","name":"Národní archiv","name_en":"National Archives","alive":true,"last_state_switch":"2023-09-27T17:30:58.648Z","state_duration":922206,"version":"5.7.2","url":"https://kramerius.nacr.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/nacr/logo","email":null,"web":"http://www.nacr.cz","street":"Archivní 4","city":"Praha 4","zip":"149 01","latitude":50.03944722,"longitude":14.49419167,"collections":2,"recommended_all":6,"recommended_public":5,"documents_all":34,"documents_public":34,"pages_all":231820,"pages_public":231820,"created_at":"2017-03-21T01:27:08.068Z","updated_at":"2023-09-27T17:30:59.408Z","last_document_at":"2023-09-07T13:13:54.120Z","last_document_before":31,"licenses":[],"model_monograph_all":13,"model_monograph_public":13,"model_periodical_all":21,"model_periodical_public":21,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":0,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":34310,"model_periodicalitem_public":34310,"model_supplement_all":1766,"model_supplement_public":1766,"model_periodicalvolume_all":221,"model_periodicalvolume_public":221,"model_monographunit_all":0,"model_monographunit_public":0,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":4,"model_internalpart_public":4,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":21,"code":"svkkl","sigla":"KLG001","name":"Středočeská vědecká knihovna v Kladně","name_en":"Central Bohemian Research Library","alive":true,"last_state_switch":"2023-09-27T13:32:05.344Z","state_duration":936539,"version":"5.3.4","url":"http://kramerius.svkkl.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/svkkl/logo","email":null,"web":"http://www.svkkl.cz","street":"tř. Gen.Klapálka 1641","city":"Kladno","zip":"272 01","latitude":50.14507778,"longitude":14.107725,"collections":1,"recommended_all":18,"recommended_public":14,"documents_all":2626,"documents_public":245,"pages_all":517650,"pages_public":79354,"created_at":"2016-10-23T13:50:30.234Z","updated_at":"2023-09-27T13:32:05.965Z","last_document_at":"2023-08-14T17:28:49.193Z","last_document_before":55,"licenses":[],"model_monograph_all":2353,"model_monograph_public":216,"model_periodical_all":256,"model_periodical_public":28,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":6,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":11,"model_sheetmusic_public":1,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":2,"model_article_public":0,"model_periodicalitem_all":12871,"model_periodicalitem_public":3798,"model_supplement_all":166,"model_supplement_public":58,"model_periodicalvolume_all":633,"model_periodicalvolume_public":229,"model_monographunit_all":9,"model_monographunit_public":3,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":569,"model_internalpart_public":138,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":499,"model_picture_public":165},{"id":30,"code":"mendelu","sigla":"BOD006","name":"Mendelova univerzita v Brně","name_en":"Mendel University in Brno","alive":true,"last_state_switch":"2023-09-27T13:06:06.900Z","state_duration":938098,"version":"5.3.2","url":"http://kramerius4.mendelu.cz","new_client_url":null,"new_client_version":null,"logo":"https://api.registr.digitalniknihovna.cz/libraries/mendelu/logo","email":null,"web":"http://uvis.mendelu.cz/","street":"Zemědělská 1","city":"Brno","zip":"613 00","latitude":49.20897778,"longitude":16.61413056,"collections":0,"recommended_all":6,"recommended_public":6,"documents_all":1131,"documents_public":516,"pages_all":218152,"pages_public":189716,"created_at":"2016-10-23T13:50:30.278Z","updated_at":"2023-09-27T13:18:27.221Z","last_document_at":"2016-10-02T18:56:38.271Z","last_document_before":2562,"licenses":[],"model_monograph_all":655,"model_monograph_public":516,"model_periodical_all":0,"model_periodical_public":0,"model_soundrecording_all":0,"model_soundrecording_public":0,"model_map_all":476,"model_map_public":0,"model_graphic_all":0,"model_graphic_public":0,"model_sheetmusic_all":0,"model_sheetmusic_public":0,"model_archive_all":0,"model_archive_public":0,"model_manuscript_all":0,"model_manuscript_public":0,"model_article_all":0,"model_article_public":0,"model_periodicalitem_all":0,"model_periodicalitem_public":0,"model_supplement_all":0,"model_supplement_public":0,"model_periodicalvolume_all":0,"model_periodicalvolume_public":0,"model_monographunit_all":122,"model_monographunit_public":115,"model_track_all":0,"model_track_public":0,"model_soundunit_all":0,"model_soundunit_public":0,"model_internalpart_all":1517,"model_internalpart_public":440,"model_convolute_all":0,"model_convolute_public":0,"model_picture_all":0,"model_picture_public":0},{"id":49,"code":"k7test","sigla":null,"name":"MZK - testovací instalace K7","name_en":"MZK - test installation of K7","alive":true,"last_state_switch":"2023-10-05T16:05:29.539Z","state_duration":236135,"version":"7.0.29","url":"https://api.kramerius.mzk.cz","new_client_url":"https://kramerius.k7-test.mzk.cz/","new_client_version":"2.5.0","logo":"https://api.registr.digitalniknihovna.cz/libraries/k7test/logo","email":null,"web":null,"street":null,"city":null,"zip":null,"latitude":null,"longitude":null,"collections":505,"recommended_all":0,"recommended_public":0,"documents_all":277851,"documents_public":56538,"pages_all":68114634,"pages_public":8413698,"created_at":"2022-06-02T12:55:59.723Z","updated_at":"2023-10-08T06:04:44.087Z","last_document_at":"2023-10-06T08:15:21.379Z","last_document_before":2,"licenses":[{"id":"dnnto","count":38000763},{"id":"onsite","count":14854347},{"id":"public","count":7953683},{"id":"dnntt","count":6876272},{"id":"mzk_public-contract","count":360735},{"id":"covid","count":277819},{"id":"mzk_public-muo","count":196776},{"id":"onsite-sheetmusic","count":20387},{"id":"license","count":2731},{"id":"public-muo","count":241}],"model_monograph_all":233953,"model_monograph_public":23933,"model_periodical_all":4255,"model_periodical_public":844,"model_soundrecording_all":683,"model_soundrecording_public":168,"model_map_all":14011,"model_map_public":9538,"model_graphic_all":18907,"model_graphic_public":18904,"model_sheetmusic_all":5143,"model_sheetmusic_public":2256,"model_archive_all":548,"model_archive_public":548,"model_manuscript_all":221,"model_manuscript_public":220,"model_article_all":81813,"model_article_public":11266,"model_periodicalitem_all":615814,"model_periodicalitem_public":238832,"model_supplement_all":73254,"model_supplement_public":52295,"model_periodicalvolume_all":34443,"model_periodicalvolume_public":9158,"model_monographunit_all":8832,"model_monographunit_public":1755,"model_track_all":5059,"model_track_public":561,"model_soundunit_all":1469,"model_soundunit_public":391,"model_internalpart_all":1316,"model_internalpart_public":1135,"model_convolute_all":130,"model_convolute_public":127,"model_picture_all":56,"model_picture_public":56}] \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml index eaa137005d..10dafb6055 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml @@ -4,6 +4,8 @@ $iteration.dl$ $iteration.dl$-full full + + $iteration.workingtime$ @@ -31,6 +33,7 @@ select + 80 $iteration.url$ @@ -45,6 +48,7 @@ $destination.url$ + select @@ -74,7 +78,7 @@ - PID root_pid dnnt-labels contains-dnnt-labels + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model $iteration.dl$ diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_1.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_1.xml new file mode 100644 index 0000000000..ca29f7bd4a --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_1.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2009-01-01T00:00:00.000Z TO 2013-01-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + cdkmzk + CCYeZPSe + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_10.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_10.xml new file mode 100644 index 0000000000..875051f8c6 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_10.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2016-06-01T00:00:00.000Z TO 2017-01-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + cdkmzk + CCYeZPSe + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_11.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_11.xml new file mode 100644 index 0000000000..7fbe78e4eb --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_11.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2016-06-01T00:00:00.000Z TO 2017-01-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + cdkmzk + CCYeZPSe + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_12.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_12.xml new file mode 100644 index 0000000000..68068838b3 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_12.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2017-01-01T00:00:00.000Z TO 2017-06-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + cdkmzk + CCYeZPSe + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_2.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_2.xml new file mode 100644 index 0000000000..127a643032 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_2.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2013-01-01T00:00:00.000Z TO 2013-06-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + cdkmzk + CCYeZPSe + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_3.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_3.xml new file mode 100644 index 0000000000..6d00838677 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_3.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2013-06-01T00:00:00.000Z TO 2014-01-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + cdkmzk + CCYeZPSe + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_4.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_4.xml new file mode 100644 index 0000000000..fbd6e3641a --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_4.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2014-01-01T00:00:00.000Z TO 2014-06-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + cdkmzk + CCYeZPSe + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_5.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_5.xml new file mode 100644 index 0000000000..deded4928c --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_5.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2014-06-01T00:00:00.000Z TO 2015-01-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + cdkmzk + CCYeZPSe + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_6.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_6.xml new file mode 100644 index 0000000000..ec5fba7b12 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_6.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2014-06-01T00:00:00.000Z TO 2015-01-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + cdkmzk + CCYeZPSe + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_7.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_7.xml new file mode 100644 index 0000000000..40ed8e2475 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_7.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2015-01-01T00:00:00.000Z TO 2015-06-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + cdkmzk + CCYeZPSe + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_8.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_8.xml new file mode 100644 index 0000000000..22936931b1 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_8.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2015-06-01T00:00:00.000Z TO 2016-01-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + cdkmzk + CCYeZPSe + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_9.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_9.xml new file mode 100644 index 0000000000..66faa01e62 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_9.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2016-01-01T00:00:00.000Z TO 2016-06-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + cdkmzk + CCYeZPSe + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/ParallelProcessImplTest.java b/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/ParallelProcessImplTest.java new file mode 100644 index 0000000000..87ae6779fc --- /dev/null +++ b/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/ParallelProcessImplTest.java @@ -0,0 +1,122 @@ +package cz.incad.kramerius.services; + +import java.time.LocalDate; +import java.time.LocalDateTime; +import java.time.Month; + +import org.apache.commons.lang3.tuple.Pair; +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + + + + + +public class ParallelProcessImplTest { + + /** Simple case */ + @Test + public void testIsWorkingTimeSimple() { + + // 22:19 - window is 08:20 - 23:00 -> true + Pair p0 = dateTime(2023, Month.OCTOBER, 6, 22, 19, 35); + Assert.assertTrue(ParallelProcessImpl.isWorkingTimeImpl("08:20", "23:00", p0.getLeft(), p0.getRight())); + + // 07:52 - window is 08:20 - 23:00 -> false + Pair p1 = dateTime(2023, Month.OCTOBER, 6, 7, 52, 35); + Assert.assertFalse(ParallelProcessImpl.isWorkingTimeImpl("08:20", "23:00", p1.getLeft(), p1.getRight())); + + // 08:25 - window is 08:20 - 23:00 -> true + Pair p2 = dateTime(2023, Month.OCTOBER, 7, 8, 25, 35); + Assert.assertTrue(ParallelProcessImpl.isWorkingTimeImpl("08:20", "23:00", p2.getLeft(), p2.getRight())); + + // 23:01 - window is 08:20 - 23:00 -> false + Pair p3 = dateTime(2023, Month.OCTOBER, 7, 23, 01, 35); + Assert.assertFalse(ParallelProcessImpl.isWorkingTimeImpl("08:20", "23:00", p3.getLeft(), p3.getRight())); + + // 22:59 - window is 08:20 - 23:00 -> true + Pair p4 = dateTime(2023, Month.OCTOBER, 7, 22, 59, 35); + Assert.assertTrue(ParallelProcessImpl.isWorkingTimeImpl("08:20", "23:00", p4.getLeft(), p4.getRight())); + } + + /** Over midnight case */ + @Test + public void testIsWorkingTimeMidnight() { + + // 22:19 - window is 22:20 - 05:00 -> false + Pair p0 = dateTime(2023, Month.OCTOBER, 6, 22, 19, 35); + Assert.assertFalse(ParallelProcessImpl.isWorkingTimeImpl("22:20", "05:00", p0.getLeft(), p0.getRight())); + + // 23:52 - window is 22:20 - 05:00 -> true + Pair p1 = dateTime(2023, Month.OCTOBER, 6, 23, 52, 35); + Assert.assertTrue(ParallelProcessImpl.isWorkingTimeImpl("22:20", "05:00", p1.getLeft(), p1.getRight())); + + // 00:00 - window is 22:20 - 05:00 -> true + Pair p2 = dateTime(2023, Month.OCTOBER, 7, 00, 00, 35); + Assert.assertTrue(ParallelProcessImpl.isWorkingTimeImpl("22:20", "05:00", p2.getLeft(), p2.getRight())); + + // 04:59 - window is 22:20 - 05:00 -> true + Pair p3 = dateTime(2023, Month.OCTOBER, 7, 04, 59, 59); + Assert.assertTrue(ParallelProcessImpl.isWorkingTimeImpl("22:20", "05:00", p3.getLeft(), p3.getRight())); + + // 05:00 - window is 22:20 - 05:00 -> false + Pair p4 = dateTime(2023, Month.OCTOBER, 7, 05, 00, 00); + Assert.assertFalse(ParallelProcessImpl.isWorkingTimeImpl("22:20", "05:00", p4.getLeft(), p4.getRight())); + + } + + /** Waiting time calculation; simple case */ + @Test + public void testWaitingTimeSimple() throws InterruptedException { + // 22:20 - window is 8:20 - 23:00 -> wait for next day; 10 hours + Pair p0 = dateTime(2023, Month.OCTOBER, 6, 22, 20, 00); + long wait0 = ParallelProcessImpl.waitUntilStartWorkingTimeImpl("08:20", "23:00", p0.getLeft(), p0.getRight()); + long toMidnight = 6000000L; + long afterMidnght = 28800000L+1200000L; + Assert.assertTrue(wait0 == (toMidnight+afterMidnght)); + Assert.assertTrue(ParallelProcessImpl.isWorkingTimeImpl("08:20", "23:00", p0.getLeft(), p0.getRight())); + + // 07:20 - window is 8:20 - 23:00 -> 1 hour + Pair p1 = dateTime(2023, Month.OCTOBER, 6, 07, 20, 00); + long wait1 = ParallelProcessImpl.waitUntilStartWorkingTimeImpl("08:20", "23:00", p1.getLeft(), p1.getRight()); + Assert.assertTrue(wait1 == 3600000L); + Assert.assertFalse(ParallelProcessImpl.isWorkingTimeImpl("08:20", "23:00", p1.getLeft(), p1.getRight())); + + + // 03:20 - window is 8:20 - 23:00 -> 5 hour + Pair p2 = dateTime(2023, Month.OCTOBER, 6, 03, 20, 00); + long wait2 = ParallelProcessImpl.waitUntilStartWorkingTimeImpl("08:20", "23:00", p2.getLeft(), p2.getRight()); + Assert.assertTrue(wait2 == 18000000L); + Assert.assertFalse(ParallelProcessImpl.isWorkingTimeImpl("08:20", "23:00", p2.getLeft(), p2.getRight())); + + } + + @Test + public void testWaitingTimeMidnight() throws InterruptedException { + // 22:19:35 neni v intervalu a ceka 25 sekund + Pair p0 = dateTime(2023, Month.OCTOBER, 6, 22, 19, 35); + long wait1 = ParallelProcessImpl.waitUntilStartWorkingTimeImpl("22:20", "05:00", p0.getLeft(), p0.getRight()); + Assert.assertTrue(wait1 == 25000); + + // 16:20:00 neni v intervalu a ceka 6 hodin + Pair p1 = dateTime(2023, Month.OCTOBER, 6, 16, 20, 00); + long wait2 = ParallelProcessImpl.waitUntilStartWorkingTimeImpl("22:20", "05:00", p1.getLeft(), p1.getRight()); + Assert.assertTrue(wait2 == 21600000); + + + // 22:30 - je v intervalu; tedy pracuje a pokud ceka, pak ceka dalsi kolo 23 hodin; 50 minut + Pair p2 = dateTime(2023, Month.OCTOBER, 6, 22, 30, 00); + long wait3 = ParallelProcessImpl.waitUntilStartWorkingTimeImpl("22:20", "05:00", p2.getLeft(), p2.getRight()); + long twentythree = 82800000L; + long fifty = 3000000L; + Assert.assertTrue(wait3 == (twentythree + fifty)); + } + + + private PairdateTime(int year, Month month, int dayOfMonth, int hour, int minutes, int seconds) { + LocalDate date = LocalDate.of(year, month, dayOfMonth); + LocalDateTime dateTime = LocalDateTime.of(year, month, dayOfMonth, hour, minutes, seconds); + return Pair.of(date, dateTime); + } +} From f5a6fdfad1dd8a889bf82ba96715e7f399bcbcf1 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 9 Oct 2023 13:00:26 +0200 Subject: [PATCH 090/235] Changed sliced xmls --- .../replicate/configurations/sliced/nkp/nkp_sliced_1.xml | 4 ++-- .../replicate/configurations/sliced/nkp/nkp_sliced_10.xml | 4 ++-- .../replicate/configurations/sliced/nkp/nkp_sliced_11.xml | 4 ++-- .../replicate/configurations/sliced/nkp/nkp_sliced_2.xml | 4 ++-- .../replicate/configurations/sliced/nkp/nkp_sliced_3.xml | 4 ++-- .../replicate/configurations/sliced/nkp/nkp_sliced_4.xml | 4 ++-- .../replicate/configurations/sliced/nkp/nkp_sliced_5.xml | 4 ++-- .../replicate/configurations/sliced/nkp/nkp_sliced_6.xml | 4 ++-- .../replicate/configurations/sliced/nkp/nkp_sliced_7.xml | 4 ++-- .../replicate/configurations/sliced/nkp/nkp_sliced_8.xml | 4 ++-- .../replicate/configurations/sliced/nkp/nkp_sliced_9.xml | 4 ++-- 11 files changed, 22 insertions(+), 22 deletions(-) diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_1.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_1.xml index ca29f7bd4a..52f7865b9a 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_1.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_1.xml @@ -42,8 +42,8 @@ 88 - cdkmzk - CCYeZPSe + $iteration.user$ + $iteration.pass$ true diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_10.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_10.xml index 875051f8c6..49dfe1776f 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_10.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_10.xml @@ -42,8 +42,8 @@ 88 - cdkmzk - CCYeZPSe + $iteration.user$ + $iteration.pass$ true diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_11.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_11.xml index 7fbe78e4eb..e77c3718bc 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_11.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_11.xml @@ -42,8 +42,8 @@ 88 - cdkmzk - CCYeZPSe + $iteration.user$ + $iteration.pass$ true diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_2.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_2.xml index 127a643032..1f83543776 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_2.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_2.xml @@ -42,8 +42,8 @@ 88 - cdkmzk - CCYeZPSe + $iteration.user$ + $iteration.pass$ true diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_3.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_3.xml index 6d00838677..4300d8703f 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_3.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_3.xml @@ -42,8 +42,8 @@ 88 - cdkmzk - CCYeZPSe + $iteration.user$ + $iteration.pass$ true diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_4.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_4.xml index fbd6e3641a..f9e28b09ec 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_4.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_4.xml @@ -17,8 +17,8 @@ 5000 CURSOR - cdkmzk - CCYeZPSe + $iteration.user$ + $iteration.pass$ modified_date diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_5.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_5.xml index deded4928c..5b873f8cb7 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_5.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_5.xml @@ -42,8 +42,8 @@ 88 - cdkmzk - CCYeZPSe + $iteration.user$ + $iteration.pass$ true diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_6.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_6.xml index ec5fba7b12..09801d19b0 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_6.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_6.xml @@ -42,8 +42,8 @@ 88 - cdkmzk - CCYeZPSe + $iteration.user$ + $iteration.pass$ true diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_7.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_7.xml index 40ed8e2475..c4eb4d3a14 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_7.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_7.xml @@ -42,8 +42,8 @@ 88 - cdkmzk - CCYeZPSe + $iteration.user$ + $iteration.pass$ true diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_8.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_8.xml index 22936931b1..cfb14a6bec 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_8.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_8.xml @@ -42,8 +42,8 @@ 88 - cdkmzk - CCYeZPSe + $iteration.user$ + $iteration.pass$ true diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_9.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_9.xml index 66faa01e62..ddcf900cc3 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_9.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_9.xml @@ -42,8 +42,8 @@ 88 - cdkmzk - CCYeZPSe + $iteration.user$ + $iteration.pass$ true From 661e89b99792f423f1c7b3b6cc2753762aeacb0d Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 14 Nov 2023 15:12:03 +0100 Subject: [PATCH 091/235] Issue https://github.com/ceskaexpedice/ceska-digitalni-knihovna/issues/49 --- .../transform/BasicSourceToDestTransform.java | 129 ++++- .../kramerius/services/utils/SolrUtils.java | 2 + .../replicate/AbstractReplicateWorker.java | 2 +- .../workers/replicate/BatchUtils.java | 4 +- .../configurations/default_k7-update.xml | 9 +- .../configurations/defult_k5-update.xml | 2 +- .../apiNew/client/v60/SearchResource.java | 5 + .../client/v60/filter/DefaultFilter.java | 247 ++++++++-- .../apiNew/client/v60/libs/Instances.java | 1 + .../client/v60/libs/PhysicalLocationMap.java | 82 ++++ .../DefaultPropertiesInstances.java | 30 +- .../client/v60/filter/DefaultFilterTest.java | 449 +++++++++++++++--- .../client/v60/filter/filter_simple.xml | 2 + 13 files changed, 824 insertions(+), 140 deletions(-) create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/PhysicalLocationMap.java diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/BasicSourceToDestTransform.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/BasicSourceToDestTransform.java index f4ae949539..c09abbb8b1 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/BasicSourceToDestTransform.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/BasicSourceToDestTransform.java @@ -17,6 +17,8 @@ import java.util.function.Consumer; import java.util.logging.Level; +import javax.xml.transform.TransformerException; + //TODO: Change it in future; copied from K5 cdk download public class BasicSourceToDestTransform extends SourceToDestTransform{ @@ -36,24 +38,22 @@ public boolean acceptElement(Element element) { - public static void simpleValue(String pid, Document feedDoc, Element feedDocElm, Node node, String derivedName, boolean dontCareAboutNonCopiingFields, CopyReplicateConsumer consumer) { + public static void simpleValue(String pid, Document feedDoc, Element feedDocElm, Node node, String derivedName, CopyReplicateConsumer consumer) { //String attributeName = ((Element)node).getAttribute("name"); String attributeName = derivedName != null ? derivedName : ((Element)node).getAttribute("name"); - if (dontCareAboutNonCopiingFields) { - - Element strElm = feedDoc.createElement("field"); - strElm.setAttribute("name", attributeName); - String content = StringEscapeUtils.escapeXml(node.getTextContent()); - // add to context to process - strElm.setTextContent(content); - - ModifyFieldResult result = ModifyFieldResult.none; - if (consumer != null) { - result = consumer.modifyField(strElm); - } - if (!result.equals(ModifyFieldResult.delete)) { - feedDocElm.appendChild(strElm); - } + + Element strElm = feedDoc.createElement("field"); + strElm.setAttribute("name", attributeName); + String content = StringEscapeUtils.escapeXml(node.getTextContent()); + // add to context to process + strElm.setTextContent(content); + + ModifyFieldResult result = ModifyFieldResult.none; + if (consumer != null) { + result = consumer.modifyField(strElm); + } + if (!result.equals(ModifyFieldResult.delete)) { + feedDocElm.appendChild(strElm); } } @@ -64,7 +64,7 @@ public static void arrayValue(String pid, Element sourceDocElement, Document fee Node n = childNodes.item(i); if (n.getNodeType() == Node.ELEMENT_NODE) { //simpleValue(feedDoc,feedDocElement, n, false); - simpleValue(pid, feedDoc,feedDocElement, n, attributeName, false, consumer); + simpleValue(pid, feedDoc,feedDocElement, n, attributeName, consumer); } } } @@ -96,15 +96,108 @@ public void transform(Element sourceDocElm, Document destDocument, Element destD if (node.getNodeType() == Node.ELEMENT_NODE) { List primitiveVals = Arrays.asList("str","int","bool", "date"); if (primitiveVals.contains(node.getNodeName())) { - simpleValue(pid, destDocument,destDocElem, node,null, false, consumer); + simpleValue(pid, destDocument,destDocElem, node,null, consumer); } else { arrayValue(pid,sourceDocElm, destDocument,destDocElem,node, consumer); } } } } +// try { +// +// XMLUtils.print(destDocElem, System.out); +// } catch (TransformerException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } + } + + // special not stored fields browse_autor, browse_title + public static void browseAuthorsAndTitles(Element sourceDocElm,Document ndoc, Element docElm) { + try { + UTFSort utf_sort = new UTFSort(); + //FedoraOperations operations = new FedoraOperations(); + + // browse author -- skip + Element browseAuthorInSource = XMLUtils.findElement(sourceDocElm, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + return element.getAttribute("name").equals("browse_autor"); + } + }); + // browse title doens't exist + if (browseAuthorInSource == null) { + Element dcCreators = XMLUtils.findElement(sourceDocElm, new XMLUtils.ElementsFilter() { + + @Override + public boolean acceptElement(Element e) { + String attribute = e.getAttribute("name"); + return attribute.equals("dc.creator"); + } + }); + if (dcCreators != null) { + List dcCreatorsStrings = XMLUtils.getElements(dcCreators); + for (Element author : dcCreatorsStrings) { + //## + String textContent = author.getTextContent(); + String prepared = utf_sort.translate(textContent)+"##"+textContent; + //String prepared = operations.prepareCzech(textContent)+"##"+textContent; + Element strElm = ndoc.createElement("field"); + strElm.setAttribute("name", "browse_autor"); + docElm.appendChild(strElm); + strElm.setTextContent(prepared); + } + } + + } + + + // browse author -- skip + Element browseTitleInSource = XMLUtils.findElement(sourceDocElm, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + return element.getAttribute("name").equals("browse_title"); + } + }); + + if (browseTitleInSource == null) { + Element model = XMLUtils.findElement(sourceDocElm, new XMLUtils.ElementsFilter() { + + @Override + public boolean acceptElement(Element e) { + String attribute = e.getAttribute("name"); + return attribute.equals("fedora.model"); + } + }); + + + Element dcTitle = XMLUtils.findElement(sourceDocElm, new XMLUtils.ElementsFilter() { + + @Override + public boolean acceptElement(Element e) { + String attribute = e.getAttribute("name"); + return attribute.equals("dc.title"); + } + }); + + if (dcTitle != null && model != null && Arrays.asList(KConfiguration.getInstance().getConfiguration().getStringArray("indexer.browseModels")).contains(model.getTextContent().trim())) { + Element strElm = ndoc.createElement("field"); + strElm.setAttribute("name", "browse_title"); + docElm.appendChild(strElm); + String textContent = dcTitle.getTextContent(); + String prepared = utf_sort.translate(textContent)+"##"+textContent; + strElm.setTextContent(prepared); + } + } + } catch (Exception e) { + BatchUtils.LOGGER.log(Level.SEVERE,e.getMessage(), e); + throw new RuntimeException(e.getMessage()); + } + } + + @Override public String getField(String fieldId) { diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/SolrUtils.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/SolrUtils.java index 88067558b4..b4a93d12c7 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/SolrUtils.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/SolrUtils.java @@ -36,6 +36,8 @@ public static String sendToDest(String destSolr, Client client, Document batchDo try { StringWriter writer = new StringWriter(); XMLUtils.print(batchDoc, writer); + + WebResource r = client.resource(destSolr); ClientResponse resp = r.accept(MediaType.TEXT_XML).type(MediaType.TEXT_XML).entity(writer.toString(), MediaType.TEXT_XML).post(ClientResponse.class); if (resp.getStatus() != ClientResponse.Status.OK.getStatusCode()) { diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java index f57eb05005..22abcc02ec 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java @@ -29,7 +29,7 @@ public abstract class AbstractReplicateWorker extends Worker { + "datum_str datum rok datum_begin datum_end datum_page issn mdt ddt dostupnost keywords " + "geographic_names collection sec model_path pid_path rels_ext_index level dc.title title_sort " + "title_sort dc.creator dc.identifier language dc.description details facet_title browse_title browse_autor img_full_mime viewable " - + "virtual location range mods.shelfLocator mods.physicalLocation text dnnt dnnt-labels"; + + "virtual location range mods.shelfLocator mods.physicalLocation text dnnt dnnt-labels contains-dnnt-labels"; public static final String DEFAULT_PID_FIELD = "PID"; public static final String COLLECTION_FIELD = "collection"; diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/BatchUtils.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/BatchUtils.java index 447d4a266f..ffa391e313 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/BatchUtils.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/BatchUtils.java @@ -13,6 +13,7 @@ import org.apache.commons.io.FileUtils; import org.w3c.dom.Document; import org.w3c.dom.Element; +import com.google.gwt.dom.client.SourceElement; import cz.incad.kramerius.service.MigrateSolrIndexException; import cz.incad.kramerius.services.transform.SourceToDestTransform; @@ -43,7 +44,8 @@ public boolean acceptElement(Element elm) { // basic transform srcTransform.transform(sourceDocElm, destBatch, destDocElement,consumer); - + + // composite id is not supported if (compositeId && root != null && child != null) { diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/default_k7-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/default_k7-update.xml index 46e808ec67..9f7c4ed864 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/default_k7-update.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/default_k7-update.xml @@ -47,10 +47,8 @@ cdk.collection - - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* titles.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors + + indexer_version pid root.pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* titles.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors @@ -72,7 +70,8 @@ - pid root.pid licenses contains_licenses licenses_of_ancestors in_collections in_collections.direct + + pid root.pid licenses contains_licenses licenses_of_ancestors titles.* collection.* in_collections in_collections.* title.* titles.* text_ocr $iteration.dl$ diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/defult_k5-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/defult_k5-update.xml index 0f929f9c7b..b49cb3c163 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/defult_k5-update.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/defult_k5-update.xml @@ -76,7 +76,7 @@ - PID root_pid dnnt-labels contains-dnnt-labels + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model $iteration.dl$ diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java index 9de55d3981..36c18cb4b8 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java @@ -23,6 +23,7 @@ import cz.incad.kramerius.rest.apiNew.client.v60.filter.ProxyFilter; import cz.incad.kramerius.rest.apiNew.exceptions.BadRequestException; import cz.incad.kramerius.rest.apiNew.exceptions.InternalErrorException; +import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.XMLUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; @@ -36,6 +37,7 @@ import org.xml.sax.SAXException; import javax.inject.Named; +import javax.inject.Provider; import javax.ws.rs.GET; import javax.ws.rs.Path; import javax.ws.rs.QueryParam; @@ -71,6 +73,9 @@ public class SearchResource { @Inject ProxyFilter proxyFilter; + @javax.inject.Inject + Provider userProvider; + @GET public Response get(@Context UriInfo uriInfo, @QueryParam("wt") String wt) { try { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java index bbbb42d0b0..3bb61c5d1b 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java @@ -1,7 +1,13 @@ package cz.incad.kramerius.rest.apiNew.client.v60.filter; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; import java.util.stream.Collectors; import org.json.JSONArray; @@ -12,6 +18,7 @@ import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.PhysicalLocationMap; import cz.incad.kramerius.rest.apiNew.client.v60.libs.properties.DefaultPropertiesInstances; import cz.incad.kramerius.utils.XMLUtils; @@ -53,49 +60,211 @@ public String enhancedFilter(String f) { - @Override - public void filterValue(Element rawDoc) { - - List dInsts = libraries.disabledInstances().stream().map(OneInstance::getName).collect(Collectors.toList()); - Element cdkElement = XMLUtils.findElement(rawDoc, new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - String name = element.getAttribute("name"); - if (name != null && name.equals("cdk.collection")) { - return true; - } - return false; - } - }); - - if (cdkElement != null) { - List elements = XMLUtils.getElements(cdkElement); - elements.forEach(e-> { - String content = e.getTextContent().trim(); - synchronized(rawDoc.getOwnerDocument()) { - if (dInsts.contains(content)) { - cdkElement.removeChild(e); - } - } - }); - } + @Override + public void filterValue(Element rawDoc) { + // physical locations + Element physicalLocationElm = XMLUtils.findElement(rawDoc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + if (name != null && name.equals("physical_locations.facet")) { + return true; + } + return false; + } + }); + + List acronymsByPhysicalLocations = physicalLocationElm != null ? acronymsSortedByPhysicalLocation(XMLUtils.getElements(physicalLocationElm), libraries) : new ArrayList<>(); - } + List disabledInstance = libraries.disabledInstances().stream().map(OneInstance::getName).collect(Collectors.toList()); + Element cdkLeaderElement = XMLUtils.findElement(rawDoc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + if (name != null && name.equals("cdk.leader")) { + return true; + } + return false; + } + }); - @Override - public void filterValue(JSONObject rawDoc) { - List dInsts = libraries.disabledInstances().stream().map(OneInstance::getName).collect(Collectors.toList()); - if (rawDoc.has("cdk.collection")) { - JSONArray col = rawDoc.getJSONArray("cdk.collection"); - JSONArray nCol = new JSONArray(); - for (int i = 0; i < col.length(); i++) { - String lib = col.getString(i); - if (!dInsts.contains(lib)) { nCol.put(lib); } - } - rawDoc.put("cdk.collection", nCol); - } + + Element cdkElement = XMLUtils.findElement(rawDoc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + if (name != null && name.equals("cdk.collection")) { + return true; + } + return false; + } + }); + + if (cdkElement != null) { + + + + List realCollections = XMLUtils.getElements(cdkElement); + List realCollectionNames = realCollections.stream().map(Element::getTextContent).collect(Collectors.toList()); + + // odebrani vsech + List elements = XMLUtils.getElements(cdkElement); + elements.forEach(e-> { + synchronized(rawDoc.getOwnerDocument()) { + cdkElement.removeChild(e); + } + }); + + if (acronymsByPhysicalLocations != null && acronymsByPhysicalLocations.size() > 0) { + + String name = "cdk.collection.sorted"; + Element sortedElement = rawDoc.getOwnerDocument().createElement("bool"); + sortedElement.setAttribute("name", name); + sortedElement.setTextContent("true"); + synchronized(rawDoc.getOwnerDocument()) { + rawDoc.appendChild(sortedElement); + } + + Map map = new HashMap<>(); + realCollections.stream().forEach(re-> { + String acronym = re.getTextContent(); + map.put(acronym, re); + }); + + synchronized(rawDoc.getOwnerDocument()) { + + acronymsByPhysicalLocations.forEach(sortedAc-> { + Element rElem = map.get(sortedAc); + if (!disabledInstance.contains(sortedAc)) { + cdkElement.appendChild(rElem); + } + realCollectionNames.remove(sortedAc); + }); + + if (!realCollectionNames.isEmpty()) { + realCollectionNames.forEach(rc-> { + Element rElem = map.get(rc); + if (!disabledInstance.contains(rc)) { + cdkElement.appendChild(rElem); + } + }); + } + + if (cdkLeaderElement != null) { + cdkLeaderElement.setTextContent(acronymsByPhysicalLocations.get(0)); + } + + } + } else { + + String name = "cdk.collection.sorted"; + Element sortedElement = rawDoc.getOwnerDocument().createElement("bool"); + sortedElement.setAttribute("name", name); + sortedElement.setTextContent("false"); + synchronized(rawDoc.getOwnerDocument()) { + rawDoc.appendChild(sortedElement); + } + + synchronized(rawDoc.getOwnerDocument()) { + elements.forEach(e-> { + String content = e.getTextContent().trim(); + if (!disabledInstance.contains(content)) { + cdkElement.appendChild(e); + } + }); + } + } + } } + private List acronymsSortedByPhysicalLocation(List locations, Instances instances) { + PhysicalLocationMap physMap = new PhysicalLocationMap(); + List textLocations = locations.stream().map(Element::getTextContent).map(String::trim).collect(Collectors.toList()); + List acronymsByPhysicalLocations = textLocations.stream().map(sigla-> { + return physMap.findBySigla(sigla); + }).map(acronym-> { + if (acronym != null) { + if (instances.isEnabledInstance(acronym)) { + return acronym; + } else { + return null; + } + } else { + return null; + } + }).filter(acronym -> acronym != null).collect(Collectors.toList()); + return acronymsByPhysicalLocations; + } + + private List acronymsSortedByPhysicalLocation(JSONArray locations, Instances instances) { + if (locations != null) { + PhysicalLocationMap physMap = new PhysicalLocationMap(); + + List textLocations = new ArrayList<>(); + for (int i = 0; i < locations.length(); i++) { textLocations.add(locations.getString(i)); } + + List acronymsByPhysicalLocations = textLocations.stream().map(sigla-> { + return physMap.findBySigla(sigla); + }).map(acronym-> { + if (acronym != null) { + if (instances.isEnabledInstance(acronym)) { + return acronym; + } else { + return null; + } + } else { + return null; + } + }).filter(acronym -> acronym != null).collect(Collectors.toList()); + return acronymsByPhysicalLocations; + } else { + return new ArrayList<>(); + } + } + + @Override + public void filterValue(JSONObject rawDoc) { + List dInsts = libraries.disabledInstances().stream().map(OneInstance::getName).collect(Collectors.toList()); + if (rawDoc.has("cdk.collection")) { + + List acronymsSortedByPhysicalLocation = acronymsSortedByPhysicalLocation(rawDoc.optJSONArray("physical_locations.facet"), libraries); + + JSONArray col = rawDoc.getJSONArray("cdk.collection"); + JSONArray nCol = new JSONArray(); + if (!acronymsSortedByPhysicalLocation.isEmpty()) { + String leader = null; + rawDoc.put("cdk.collection.sorted", true); + + List collectionFromIndex = new ArrayList<>(); + for (int i = 0; i < col.length(); i++) { collectionFromIndex.add(col.getString(i)); } + + while(!acronymsSortedByPhysicalLocation.isEmpty()) { + String acronym = acronymsSortedByPhysicalLocation.remove(0); + if (leader == null) leader = acronym; + nCol.put(acronym); + collectionFromIndex.remove(acronym); + } + + for (int i = 0; i < collectionFromIndex.size(); i++) { + if (!dInsts.contains(collectionFromIndex.get(i))) nCol.put(collectionFromIndex.get(i)); + } + + if (leader != null) { + rawDoc.put("cdk.leader", leader); + } + + } else { + rawDoc.put("cdk.collection.sorted", false); + + for (int i = 0; i < col.length(); i++) { + String lib = col.getString(i); + if (!dInsts.contains(lib)) { nCol.put(lib); } + } + } + rawDoc.put("cdk.collection", nCol); + } + } + @Override public String enhanceFacetsTerms() { List dInsts = libraries.disabledInstances().stream().map(OneInstance::getName).collect(Collectors.toList()); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/Instances.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/Instances.java index 0d528c0ba1..3394486ccd 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/Instances.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/Instances.java @@ -16,4 +16,5 @@ public interface Instances { public void cronRefresh(); + public boolean isEnabledInstance(String acronym); } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/PhysicalLocationMap.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/PhysicalLocationMap.java new file mode 100644 index 0000000000..f046d39cb2 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/PhysicalLocationMap.java @@ -0,0 +1,82 @@ +package cz.incad.kramerius.rest.apiNew.client.v60.libs; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class PhysicalLocationMap { + + private static final Map> MAPPING = new HashMap<>(); + + static { + MAPPING.put("nkp", Arrays.asList( + "ABA001", + "ABA000", + "ABA004")); + + MAPPING.put("knav", Arrays.asList( + "ABA007", + "ABB045", + "ABB030", + "ABE459", + "ABB060" + )); + + MAPPING.put("mzk", Arrays.asList( + "BOA001", + "KME450", + "BOE950", + "BOE801", + "BVE301", + "HOE802" + )); + + MAPPING.put("svkhk", Arrays.asList( + "HKA001", + "JCE301", + "RKE801", + "HKE302", + "NAE802", + "HKG001", + "TUE301" + )); + + MAPPING.put("uzei", Arrays.asList( + "ABA009")); + + MAPPING.put("kkp", Arrays.asList( + "PAG001")); + + MAPPING.put("svkul", Arrays.asList( + "ULG001")); + + MAPPING.put("nfa", Arrays.asList( + "ABC135")); + + MAPPING.put("mlp", Arrays.asList( + "ABG001")); + + MAPPING.put("nm", Arrays.asList( + "ABA010")); + + MAPPING.put("cbvk", Arrays.asList( + "CBA001")); + + MAPPING.put("vkol", Arrays.asList( + "OLA001")); + } + + public PhysicalLocationMap() {} + + + public String findBySigla(String sigla) { + for (String key : MAPPING.keySet()) { + if (MAPPING.get(key).contains(sigla)) { + return key; + } + } + return null; + } + +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java index 5f6c8eee57..e1f04dfbe8 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java @@ -8,6 +8,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; @@ -44,7 +45,7 @@ public DefaultPropertiesInstances() { refreshingConfiguration(); } - private void refreshingConfiguration() { + protected void refreshingConfiguration() { Configuration configuration = KConfiguration.getInstance().getConfiguration(); Iterator keys = configuration.getKeys("cdk.collections.sources"); while (keys.hasNext()) { @@ -54,16 +55,21 @@ private void refreshingConfiguration() { if (rest.lastIndexOf(".") > 0) { String acronym = rest.substring(0, rest.lastIndexOf(".")); if (!names.contains(acronym)) { - LOGGER.info(String.format("Adding library %s", acronym)); - names.add(acronym); - DefaultOnePropertiesInstance di = new DefaultOnePropertiesInstance(this, acronym); - instances.add(di); + addOneInstance(acronym); } } } } } + private void addOneInstance(String acronym) { + LOGGER.info(String.format("Adding library %s", acronym)); + names.add(acronym); + DefaultOnePropertiesInstance di = new DefaultOnePropertiesInstance(this, acronym); + instances.add(di); + } + + @Override public List allInstances() { this.refreshingConfiguration(); @@ -105,6 +111,20 @@ public OneInstance find(String acronym) { } } + + + + @Override + public boolean isEnabledInstance(String acronym) { + this.refreshingConfiguration(); + Optional found = instances.stream().filter(instance -> instance.getName().equals(acronym)).findFirst(); + if (found.isPresent()) { + return found.get().isConnected(); + } else { + return false; + } + } + @Override public void cronRefresh() { this.refreshingConfiguration(); diff --git a/rest/src/test/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilterTest.java b/rest/src/test/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilterTest.java index 1a522ebb40..c090416e8f 100644 --- a/rest/src/test/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilterTest.java +++ b/rest/src/test/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilterTest.java @@ -1,104 +1,413 @@ package cz.incad.kramerius.rest.apiNew.client.v60.filter; +import static org.easymock.EasyMock.createMockBuilder; + import java.io.IOException; import java.io.InputStream; import java.io.StringReader; import java.io.UnsupportedEncodingException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Iterator; import java.util.List; import java.util.Stack; +import java.util.stream.Collectors; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; import org.apache.commons.io.IOUtils; +import org.easymock.EasyMock; import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; import org.junit.Assert; -import org.junit.Ignore; import org.junit.Test; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.xml.sax.SAXException; -import cz.incad.kramerius.rest.api.k5.client.JSONDecorator; -import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance.TypeOfChangedStatus; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.properties.DefaultOnePropertiesInstance; import cz.incad.kramerius.rest.apiNew.client.v60.libs.properties.DefaultPropertiesInstances; import cz.incad.kramerius.utils.XMLUtils; -@Ignore + public class DefaultFilterTest { - @Test - public void testFilterNew() { - Instances insts = new DefaultPropertiesInstances(); - insts.find("mzk").setConnected(false, TypeOfChangedStatus.user); - //insts.setStatus("mzk", false); - ProxyFilter pf = new DefaultFilter(insts); - String newFilter = pf.newFilter(); - Assert.assertEquals("cdk.collection:(inovatika OR knav OR knav-test)",newFilter); - } - - @Test - public void testFilterApply() { - Instances insts = new DefaultPropertiesInstances(); - insts.find("mzk").setConnected(false, TypeOfChangedStatus.user); - ProxyFilter pf = new DefaultFilter(insts); - String eFilter = pf.enhancedFilter("model:monograph AND titles.search:*"); - Assert.assertEquals("model:monograph AND titles.search:* AND cdk.collection:(inovatika OR knav)",eFilter); - } - - @Test - public void testValueDocXML() throws IOException, ParserConfigurationException, SAXException, TransformerException { - Instances insts = new DefaultPropertiesInstances(); - insts.find("mzk").setConnected(false, TypeOfChangedStatus.user); - ProxyFilter pf = new DefaultFilter(insts); - - InputStream stream = DefaultFilterTest.class.getResourceAsStream("filter_simple.xml"); - String xml = IOUtils.toString(stream,"UTF-8"); - List respXML = respXML(xml); - for (Element doc : respXML) { - pf.filterValue(doc); - } - } - - @Test - public void testValueDocJSON() throws IOException { - - Instances insts = new DefaultPropertiesInstances(); - insts.find("mzk").setConnected(false, TypeOfChangedStatus.user); - ProxyFilter pf = new DefaultFilter(insts); - - InputStream stream = DefaultFilterTest.class.getResourceAsStream("filter_simple.json"); - String json = IOUtils.toString(stream,"UTF-8"); - List arrs = respJSON(json); - for (JSONArray oneArray : arrs) { - for (int i = 0; i < oneArray.length(); i++) { - JSONObject doc = oneArray.getJSONObject(i); - pf.filterValue(doc); - JSONArray cdkCol = doc.getJSONArray("cdk.collection"); - System.out.println(cdkCol); - Assert.assertTrue(cdkCol.length() == 1); - } - } - - stream = DefaultFilterTest.class.getResourceAsStream("filter_groups.json"); - json = IOUtils.toString(stream,"UTF-8"); - arrs = respJSON(json); - for (JSONArray oneArray : arrs) { - for (int i = 0; i < oneArray.length(); i++) { - JSONObject doc = oneArray.getJSONObject(i); - pf.filterValue(doc); - JSONArray cdkCol = doc.getJSONArray("cdk.collection"); - Assert.assertTrue(cdkCol.length() == 1); - } - } - - } +// @Test +// public void testFilterNew() { +// DefaultPropertiesInstances inst = createMockBuilder(DefaultPropertiesInstances.class) +// .withConstructor() +// .addMockedMethod("allInstances") +// .addMockedMethod("enabledInstances") +// .addMockedMethod("disabledInstances") +// .addMockedMethod("find") +// .addMockedMethod("isAnyDisabled") +// .createMock(); +// +// +// DefaultOnePropertiesInstance mzk = new DefaultOnePropertiesInstance(inst,"mzk"); +// DefaultOnePropertiesInstance nkp = new DefaultOnePropertiesInstance(inst,"nkp"); +// DefaultOnePropertiesInstance knav = new DefaultOnePropertiesInstance(inst,"knav"); +// knav.setConnected(true, TypeOfChangedStatus.user); +// +// +// List allInstances = new ArrayList<>(); +// allInstances.add(mzk); +// allInstances.add(nkp); +// allInstances.add(knav); +// +// +// EasyMock.expect(inst.allInstances()).andReturn(allInstances).anyTimes(); +// EasyMock.expect(inst.enabledInstances()).andReturn(allInstances).anyTimes(); +// +// EasyMock.expect(inst.find("mzk")).andReturn(mzk).anyTimes(); +// EasyMock.expect(inst.find("nkp")).andReturn(nkp).anyTimes(); +// EasyMock.expect(inst.find("knav")).andReturn(knav).anyTimes(); +// EasyMock.expect(inst.isAnyDisabled()).andReturn(true).anyTimes(); +// +// EasyMock.replay(inst); +// +// ProxyFilter pf = new DefaultFilter(inst); +// String newFilter = pf.newFilter(); +// Assert.assertEquals("cdk.collection:(mzk OR nkp OR knav)",newFilter); +// } +// +// @Test +// public void testFilterApply() { +// DefaultPropertiesInstances inst = createMockBuilder(DefaultPropertiesInstances.class) +// .withConstructor() +// .addMockedMethod("allInstances") +// .addMockedMethod("enabledInstances") +// .addMockedMethod("disabledInstances") +// .addMockedMethod("find") +// .addMockedMethod("isAnyDisabled") +// .createMock(); +// +// +// DefaultOnePropertiesInstance mzk = new DefaultOnePropertiesInstance(inst,"mzk"); +// DefaultOnePropertiesInstance nkp = new DefaultOnePropertiesInstance(inst,"nkp"); +// DefaultOnePropertiesInstance knav = new DefaultOnePropertiesInstance(inst,"knav"); +// knav.setConnected(true, TypeOfChangedStatus.user); +// +// +// List allInstances = new ArrayList<>(); +// allInstances.add(mzk); +// allInstances.add(nkp); +// allInstances.add(knav); +// +// +// EasyMock.expect(inst.allInstances()).andReturn(allInstances).anyTimes(); +// EasyMock.expect(inst.enabledInstances()).andReturn(allInstances).anyTimes(); +// +// EasyMock.expect(inst.find("mzk")).andReturn(mzk).anyTimes(); +// EasyMock.expect(inst.find("nkp")).andReturn(nkp).anyTimes(); +// EasyMock.expect(inst.find("knav")).andReturn(knav).anyTimes(); +// EasyMock.expect(inst.isAnyDisabled()).andReturn(true).anyTimes(); +// +// EasyMock.replay(inst); +// +// ProxyFilter pf = new DefaultFilter(inst); +// +// +// String eFilter = pf.enhancedFilter("model:monograph AND titles.search:*"); +// Assert.assertEquals("model:monograph AND titles.search:* AND cdk.collection:(mzk OR nkp OR knav)",eFilter); +// } +// +// +// @Test +// public void testValueDocXML() throws IOException, ParserConfigurationException, SAXException, TransformerException { +// +// DefaultPropertiesInstances inst = createMockBuilder(DefaultPropertiesInstances.class) +// .withConstructor() +// .addMockedMethod("allInstances") +// .addMockedMethod("enabledInstances") +// .addMockedMethod("disabledInstances") +// .addMockedMethod("find") +// .addMockedMethod("isAnyDisabled") +// .createMock(); +// +// +// DefaultOnePropertiesInstance mzk = new DefaultOnePropertiesInstance(inst,"mzk"); +// DefaultOnePropertiesInstance nkp = new DefaultOnePropertiesInstance(inst,"nkp"); +// DefaultOnePropertiesInstance knav = new DefaultOnePropertiesInstance(inst,"knav"); +// knav.setConnected(false, TypeOfChangedStatus.user); +// +// +// List allInstances = new ArrayList<>(); +// allInstances.add(mzk); +// allInstances.add(nkp); +// allInstances.add(knav); +// +// List enabledInstances = new ArrayList<>(); +// enabledInstances.add(nkp); +// enabledInstances.add(mzk); +// +// +// List disabledInstances = new ArrayList<>(); +// disabledInstances.add(knav); +// +// +// EasyMock.expect(inst.allInstances()).andReturn(allInstances).anyTimes(); +// EasyMock.expect(inst.enabledInstances()).andReturn(enabledInstances).anyTimes(); +// EasyMock.expect(inst.disabledInstances()).andReturn(disabledInstances).anyTimes(); +// +// EasyMock.expect(inst.find("mzk")).andReturn(mzk).anyTimes(); +// EasyMock.expect(inst.find("nkp")).andReturn(nkp).anyTimes(); +// EasyMock.expect(inst.find("knav")).andReturn(knav).anyTimes(); +// EasyMock.expect(inst.isAnyDisabled()).andReturn(true).anyTimes(); +// +// EasyMock.replay(inst); +// +// +// ProxyFilter pf = new DefaultFilter(inst); +// +// InputStream stream = DefaultFilterTest.class.getResourceAsStream("filter_simple.xml"); +// String xml = IOUtils.toString(stream,"UTF-8"); +// List respXML = respXML(xml); +// Assert.assertTrue(respXML.size() == 1); +// +// Assert.assertTrue(cdkCollectionFromDoc(respXML).equals(Arrays.asList("mzk", "nkp", "knav"))); +// +// for (Element doc : respXML) { +// pf.filterValue(doc); +// +// XMLUtils.print(doc, System.out); +// +// Assert.assertTrue(cdkCollectionFromDoc(respXML).equals(Arrays.asList("mzk", "nkp"))); +// } +// } + + +// @Test +// public void testValueDocXMLWithPhysical() throws IOException, ParserConfigurationException, SAXException, TransformerException { +// +// DefaultPropertiesInstances inst = createMockBuilder(DefaultPropertiesInstances.class) +// .withConstructor() +// .addMockedMethod("allInstances") +// .addMockedMethod("enabledInstances") +// .addMockedMethod("disabledInstances") +// .addMockedMethod("find") +// .addMockedMethod("isAnyDisabled") +// .createMock(); +// +// +// DefaultOnePropertiesInstance mzk = new DefaultOnePropertiesInstance(inst,"mzk"); +// DefaultOnePropertiesInstance nkp = new DefaultOnePropertiesInstance(inst,"nkp"); +// DefaultOnePropertiesInstance knav = new DefaultOnePropertiesInstance(inst,"knav"); +// knav.setConnected(false, TypeOfChangedStatus.user); +// +// +// List allInstances = new ArrayList<>(); +// allInstances.add(mzk); +// allInstances.add(nkp); +// allInstances.add(knav); +// +// List enabledInstances = new ArrayList<>(); +// enabledInstances.add(nkp); +// enabledInstances.add(mzk); +// +// +// List disabledInstances = new ArrayList<>(); +// disabledInstances.add(knav); +// +// +// EasyMock.expect(inst.allInstances()).andReturn(allInstances).anyTimes(); +// EasyMock.expect(inst.enabledInstances()).andReturn(enabledInstances).anyTimes(); +// EasyMock.expect(inst.disabledInstances()).andReturn(disabledInstances).anyTimes(); +// +// EasyMock.expect(inst.find("mzk")).andReturn(mzk).anyTimes(); +// EasyMock.expect(inst.find("nkp")).andReturn(nkp).anyTimes(); +// EasyMock.expect(inst.find("knav")).andReturn(knav).anyTimes(); +// EasyMock.expect(inst.isAnyDisabled()).andReturn(true).anyTimes(); +// +// EasyMock.replay(inst); +// +// +// ProxyFilter pf = new DefaultFilter(inst); +// +// InputStream stream = DefaultFilterTest.class.getResourceAsStream("filter_simple_physicalfacets.xml"); +// String xml = IOUtils.toString(stream,"UTF-8"); +// List respXML = respXML(xml); +// Assert.assertTrue(respXML.size() == 1); +// +// Assert.assertTrue(cdkCollectionFromDoc(respXML).equals(Arrays.asList("mzk", "nkp", "knav"))); +// +// for (Element doc : respXML) { +// pf.filterValue(doc); +// XMLUtils.print(doc, System.out); +// +// //Assert.assertTrue(cdkCollectionFromDoc(respXML).equals(Arrays.asList("mzk", "nkp"))); +// } +// } + + private List cdkCollectionFromDoc(List respXML) { + Element cdkCollection = XMLUtils.findElement(respXML.get(0), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("arr") && element.getAttribute("name").equals("cdk.collection")) { + return true; + } else return false; + } + }); + + Assert.assertTrue(cdkCollection != null); + List list = XMLUtils.getElements(cdkCollection).stream().map(Element::getTextContent).collect(Collectors.toList()); + return list; + } + + +// @Test +// public void testValueDocJSON() throws IOException { +// DefaultPropertiesInstances inst = createMockBuilder(DefaultPropertiesInstances.class) +// .withConstructor() +// .addMockedMethod("allInstances") +// .addMockedMethod("enabledInstances") +// .addMockedMethod("disabledInstances") +// .addMockedMethod("find") +// .addMockedMethod("isAnyDisabled") +// .createMock(); +// +// +// DefaultOnePropertiesInstance mzk = new DefaultOnePropertiesInstance(inst,"mzk"); +// DefaultOnePropertiesInstance nkp = new DefaultOnePropertiesInstance(inst,"nkp"); +// DefaultOnePropertiesInstance knav = new DefaultOnePropertiesInstance(inst,"knav"); +// knav.setConnected(false, TypeOfChangedStatus.user); +// +// +// List allInstances = new ArrayList<>(); +// allInstances.add(mzk); +// allInstances.add(nkp); +// allInstances.add(knav); +// +// List enabledInstances = new ArrayList<>(); +// enabledInstances.add(nkp); +// enabledInstances.add(mzk); +// +// +// List disabledInstances = new ArrayList<>(); +// disabledInstances.add(knav); +// +// EasyMock.expect(inst.allInstances()).andReturn(allInstances).anyTimes(); +// EasyMock.expect(inst.enabledInstances()).andReturn(enabledInstances).anyTimes(); +// EasyMock.expect(inst.disabledInstances()).andReturn(disabledInstances).anyTimes(); +// +// EasyMock.expect(inst.find("mzk")).andReturn(mzk).anyTimes(); +// EasyMock.expect(inst.find("nkp")).andReturn(nkp).anyTimes(); +// EasyMock.expect(inst.find("knav")).andReturn(knav).anyTimes(); +// EasyMock.expect(inst.isAnyDisabled()).andReturn(true).anyTimes(); +// +// EasyMock.replay(inst); +// +// ProxyFilter pf = new DefaultFilter(inst); +// +// InputStream stream = DefaultFilterTest.class.getResourceAsStream("filter_simple.json"); +// String json = IOUtils.toString(stream,"UTF-8"); +// List arrs = respJSON(json); +// for (JSONArray oneArray : arrs) { +// for (int i = 0; i < oneArray.length(); i++) { +// JSONObject doc = oneArray.getJSONObject(i); +// pf.filterValue(doc); +// JSONArray cdkCol = doc.getJSONArray("cdk.collection"); +// } +// } +// +// stream = DefaultFilterTest.class.getResourceAsStream("filter_groups.json"); +// json = IOUtils.toString(stream,"UTF-8"); +// arrs = respJSON(json); +// for (JSONArray oneArray : arrs) { +// for (int i = 0; i < oneArray.length(); i++) { +// JSONObject doc = oneArray.getJSONObject(i); +// pf.filterValue(doc); +// JSONArray cdkCol = doc.getJSONArray("cdk.collection"); +// if (cdkCol.length() == 1) { +// Assert.assertTrue(cdkCol.getString(0).equals("mzk")); +// } else { +// Assert.assertTrue(cdkCol.length() == 2 ); +// Assert.assertTrue(cdkCol.getString(0).equals("mzk")); +// } +// } +// } +// } + + + @Test + public void testValueDocWithPhysicalLocationJSON() throws IOException { + DefaultPropertiesInstances inst = createMockBuilder(DefaultPropertiesInstances.class) + .withConstructor() + .addMockedMethod("allInstances") + .addMockedMethod("enabledInstances") + .addMockedMethod("disabledInstances") + .addMockedMethod("find") + .addMockedMethod("isAnyDisabled") + .createMock(); + + + DefaultOnePropertiesInstance mzk = new DefaultOnePropertiesInstance(inst,"mzk"); + DefaultOnePropertiesInstance nkp = new DefaultOnePropertiesInstance(inst,"nkp"); + DefaultOnePropertiesInstance knav = new DefaultOnePropertiesInstance(inst,"knav"); + knav.setConnected(false, TypeOfChangedStatus.user); + + + List allInstances = new ArrayList<>(); + allInstances.add(mzk); + allInstances.add(nkp); + allInstances.add(knav); + + List enabledInstances = new ArrayList<>(); + enabledInstances.add(nkp); + enabledInstances.add(knav); + + + List disabledInstances = new ArrayList<>(); + disabledInstances.add(mzk); + + EasyMock.expect(inst.allInstances()).andReturn(allInstances).anyTimes(); + EasyMock.expect(inst.enabledInstances()).andReturn(enabledInstances).anyTimes(); + EasyMock.expect(inst.disabledInstances()).andReturn(disabledInstances).anyTimes(); + + EasyMock.expect(inst.find("mzk")).andReturn(mzk).anyTimes(); + EasyMock.expect(inst.find("nkp")).andReturn(nkp).anyTimes(); + EasyMock.expect(inst.find("knav")).andReturn(knav).anyTimes(); + EasyMock.expect(inst.isAnyDisabled()).andReturn(true).anyTimes(); + + EasyMock.replay(inst); + + ProxyFilter pf = new DefaultFilter(inst); + + InputStream stream = DefaultFilterTest.class.getResourceAsStream("filter_simple_physicalfacets.json"); + String json = IOUtils.toString(stream,"UTF-8"); + List arrs = respJSON(json); + for (JSONArray oneArray : arrs) { + for (int i = 0; i < oneArray.length(); i++) { + JSONObject doc = oneArray.getJSONObject(i); + pf.filterValue(doc); + JSONArray cdkCol = doc.getJSONArray("cdk.collection"); + //Assert.assertTrue(cdkCol); + + } + } + +// stream = DefaultFilterTest.class.getResourceAsStream("filter_groups.json"); +// json = IOUtils.toString(stream,"UTF-8"); +// arrs = respJSON(json); +// for (JSONArray oneArray : arrs) { +// for (int i = 0; i < oneArray.length(); i++) { +// JSONObject doc = oneArray.getJSONObject(i); +// pf.filterValue(doc); +// JSONArray cdkCol = doc.getJSONArray("cdk.collection"); +// if (cdkCol.length() == 1) { +// Assert.assertTrue(cdkCol.getString(0).equals("mzk")); +// } else { +// Assert.assertTrue(cdkCol.length() == 2 ); +// Assert.assertTrue(cdkCol.getString(0).equals("mzk")); +// } +// } +// } + } private List respXML(String rawString) throws ParserConfigurationException, SAXException, IOException { Document doc = XMLUtils.parseDocument(new StringReader(rawString)); diff --git a/rest/src/test/resources/cz/incad/kramerius/rest/apiNew/client/v60/filter/filter_simple.xml b/rest/src/test/resources/cz/incad/kramerius/rest/apiNew/client/v60/filter/filter_simple.xml index 4f03c6f6f4..76a749c96a 100644 --- a/rest/src/test/resources/cz/incad/kramerius/rest/apiNew/client/v60/filter/filter_simple.xml +++ b/rest/src/test/resources/cz/incad/kramerius/rest/apiNew/client/v60/filter/filter_simple.xml @@ -69,8 +69,10 @@ 6 uuid:e8686650-d81d-11ec-a96b-005056827e52!uuid:87f93b08-e35d-410c-a794-b8fe52cc6400 + mzk + nkp knav mzk From cafbbcfa442d23a64b67c4146d12deba83db884b Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 15 Nov 2023 14:05:59 +0100 Subject: [PATCH 092/235] Issue https://github.com/ceskaexpedice/ceska-digitalni-knihovna/issues/49 --- .../replicate/configurations/default_k5.xml | 3 +- .../replicate/configurations/default_k7.xml | 2 +- .../v10/proxy/ConnectedInfoResource.java | 14 + .../apiNew/client/v60/SearchResource.java | 2 - .../client/v60/filter/DefaultFilter.java | 246 ++++++++++++------ .../client/v60/libs/PhysicalLocationMap.java | 10 +- .../filter/filter_simple_physicalfacets.json | 85 ++++++ .../filter/filter_simple_physicalfacets.xml | 87 +++++++ 8 files changed, 366 insertions(+), 83 deletions(-) create mode 100644 rest/src/test/resources/cz/incad/kramerius/rest/apiNew/client/v60/filter/filter_simple_physicalfacets.json create mode 100644 rest/src/test/resources/cz/incad/kramerius/rest/apiNew/client/v60/filter/filter_simple_physicalfacets.xml diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml index 10dafb6055..2b347c976b 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml @@ -77,9 +77,8 @@ - PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model - + $iteration.dl$ diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml index 6ade81cdee..ee02a4eb2a 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml @@ -39,7 +39,7 @@ cdk.collection - indexer_version pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* titles.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors + indexer_version pid root.pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* titles.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java index bfd84be18b..4f7c6ffa98 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java @@ -29,6 +29,7 @@ import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance.TypeOfChangedStatus; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.PhysicalLocationMap; import cz.incad.kramerius.rest.apiNew.exceptions.ForbiddenException; import cz.incad.kramerius.rest.apiNew.exceptions.InternalErrorException; import cz.incad.kramerius.security.Role; @@ -77,6 +78,19 @@ public Response library(@PathParam("library") String library) { return Response.status(Response.Status.BAD_REQUEST).build(); } } + + // RESOLVED ?? + + @GET + @Path("{library}/associations") + @Produces(MediaType.APPLICATION_JSON) + public Response assocations(@PathParam("library") String library) { + PhysicalLocationMap locationMap = new PhysicalLocationMap(); + List assocations = locationMap.getAssocations(library); + JSONArray restArr = new JSONArray(); + assocations.stream().forEach(restArr::put); + return Response.ok(restArr).build(); + } @PUT @Path("{library}/status") diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java index 36c18cb4b8..fb95fb8cd1 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java @@ -263,7 +263,6 @@ public boolean acceptElement(Element element) { */ private JSONObject buildJsonFromRawSolrResponse(String rawString, String context, List decs) throws UnsupportedEncodingException, JSONException { List docsArrays = new ArrayList(); - JSONObject resultJSONObject = new JSONObject(rawString); Stack prcStack = new Stack(); prcStack.push(resultJSONObject); @@ -300,7 +299,6 @@ private JSONObject buildJsonFromRawSolrResponse(String rawString, String context filterOutFieldsFromJSON(docJSON); // decorators applyDecorators(context, decs, docJSON); - this.proxyFilter.filterValue(docJSON); } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java index 3bb61c5d1b..f33017a4b7 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java @@ -1,6 +1,10 @@ package cz.incad.kramerius.rest.apiNew.client.v60.filter; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.Iterator; @@ -8,75 +12,85 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; +import javax.inject.Named; +import javax.ws.rs.core.UriBuilder; + import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import org.w3c.dom.Element; import com.google.inject.Inject; +import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.impl.SolrAccessImplNewIndex; +import cz.incad.kramerius.rest.apiNew.client.v60.SearchResource; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; import cz.incad.kramerius.rest.apiNew.client.v60.libs.PhysicalLocationMap; import cz.incad.kramerius.rest.apiNew.client.v60.libs.properties.DefaultPropertiesInstances; import cz.incad.kramerius.utils.XMLUtils; +import cz.incad.kramerius.utils.conf.KConfiguration; public class DefaultFilter implements ProxyFilter{ - - private Instances libraries; - - @Inject - public DefaultFilter(Instances libraries) { - super(); - this.libraries = libraries; - } - @Override - public String newFilter() { - if (this.libraries.isAnyDisabled()) { - return filter(); - } else return null; - } + + public static final Logger LOGGER = Logger.getLogger(DefaultFilter.class.getName()); + + + private Instances libraries; - - private String filter() { - if (this.libraries.isAnyDisabled()) { - List eInsts = libraries.enabledInstances().stream().map(OneInstance::getName).collect(Collectors.toList()); - //List eInsts = libraries.enabledInstances().stream().map(OneInstance::getName)::getName).; - String enabled = eInsts.stream().collect(Collectors.joining(" OR ")); - return "cdk.collection:("+enabled+")"; - } else { - return null; - } - } + private SolrAccess solrAccess; + + @Inject + public DefaultFilter(Instances libraries, @Named("new-index") SolrAccess solrAccess) { + super(); + this.libraries = libraries; + this.solrAccess = solrAccess; + } - @Override - public String enhancedFilter(String f) { - if (this.libraries.isAnyDisabled()) { - return f+" AND "+filter(); - } else return f; - } - - - @Override - public void filterValue(Element rawDoc) { - // physical locations - Element physicalLocationElm = XMLUtils.findElement(rawDoc, new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - String name = element.getAttribute("name"); - if (name != null && name.equals("physical_locations.facet")) { - return true; - } - return false; - } - }); - - List acronymsByPhysicalLocations = physicalLocationElm != null ? acronymsSortedByPhysicalLocation(XMLUtils.getElements(physicalLocationElm), libraries) : new ArrayList<>(); + public String newFilter() { + if (this.libraries.isAnyDisabled()) { + return filter(); + } else + return null; + } + + private String filter() { + if (this.libraries.isAnyDisabled()) { + List eInsts = libraries.enabledInstances().stream().map(OneInstance::getName) + .collect(Collectors.toList()); + // List eInsts = + // libraries.enabledInstances().stream().map(OneInstance::getName)::getName).; + String enabled = eInsts.stream().collect(Collectors.joining(" OR ")); + return "cdk.collection:(" + enabled + ")"; + } else { + return null; + } + } + + @Override + public String enhancedFilter(String f) { + if (this.libraries.isAnyDisabled()) { + return f + " AND " + filter(); + } else + return f; + } + + + @Override + public void filterValue(Element rawDoc) { + // find all acronyms translated from physical locations + List acronymsByPhysicalLocations = acronymsSortedByPhysicalLocation(rawDoc, libraries); + // list of disabled instances List disabledInstance = libraries.disabledInstances().stream().map(OneInstance::getName).collect(Collectors.toList()); + // cdk leader element Element cdkLeaderElement = XMLUtils.findElement(rawDoc, new XMLUtils.ElementsFilter() { @Override public boolean acceptElement(Element element) { @@ -88,7 +102,7 @@ public boolean acceptElement(Element element) { } }); - + // all cdk libraries element Element cdkElement = XMLUtils.findElement(rawDoc, new XMLUtils.ElementsFilter() { @Override public boolean acceptElement(Element element) { @@ -101,8 +115,6 @@ public boolean acceptElement(Element element) { }); if (cdkElement != null) { - - List realCollections = XMLUtils.getElements(cdkElement); List realCollectionNames = realCollections.stream().map(Element::getTextContent).collect(Collectors.toList()); @@ -177,10 +189,22 @@ public boolean acceptElement(Element element) { } } - private List acronymsSortedByPhysicalLocation(List locations, Instances instances) { + private List acronymsSortedByPhysicalLocation(Element rawDoc, Instances instances) { PhysicalLocationMap physMap = new PhysicalLocationMap(); - List textLocations = locations.stream().map(Element::getTextContent).map(String::trim).collect(Collectors.toList()); - List acronymsByPhysicalLocations = textLocations.stream().map(sigla-> { + + String model = findSingleElementByName(rawDoc,"model"); + String rootPid = findSingleElementByName(rawDoc,"root.pid"); + List textLocations = findArrayElementsByName(rawDoc, "physical_locations.facet"); + + if (textLocations.size() == 0) { + List list = KConfiguration.getInstance().getConfiguration().getList("cdk.inferring_physical_locations.model" , Arrays.asList("periodicalitem","monographunit")).stream().map(Object::toString).collect(Collectors.toList()); + if (list.contains(model)) { + textLocations = inferPhysicalLocationFromPredecessor(solrAccess, rootPid); + } + return new ArrayList<>(); + } + + return textLocations.stream().map(sigla-> { return physMap.findBySigla(sigla); }).map(acronym-> { if (acronym != null) { @@ -193,33 +217,76 @@ private List acronymsSortedByPhysicalLocation(List locations, I return null; } }).filter(acronym -> acronym != null).collect(Collectors.toList()); - return acronymsByPhysicalLocations; + } - private List acronymsSortedByPhysicalLocation(JSONArray locations, Instances instances) { - if (locations != null) { - PhysicalLocationMap physMap = new PhysicalLocationMap(); - - List textLocations = new ArrayList<>(); - for (int i = 0; i < locations.length(); i++) { textLocations.add(locations.getString(i)); } - - List acronymsByPhysicalLocations = textLocations.stream().map(sigla-> { - return physMap.findBySigla(sigla); - }).map(acronym-> { - if (acronym != null) { - if (instances.isEnabledInstance(acronym)) { - return acronym; - } else { - return null; - } - } else { - return null; + private String findSingleElementByName(Element rawDoc, String name) { + Element elm = XMLUtils.findElement(rawDoc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + if (name != null && name.equals(name)) { + return true; + } + return false; + } + }); + return elm != null ? elm.getTextContent().trim() : null; + } + + + private List findArrayElementsByName(Element rawDoc, String name) { + Element elm = XMLUtils.findElement(rawDoc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + if (name != null && name.equals(name)) { + return true; } - }).filter(acronym -> acronym != null).collect(Collectors.toList()); - return acronymsByPhysicalLocations; + return false; + } + }); + if (elm != null) { + List elements = XMLUtils.getElements(elm); + return elements.stream().map(Element::getTextContent).map(String::trim).collect(Collectors.toList()); + + } else return new ArrayList<>(); + } + + + + + private List acronymsSortedByPhysicalLocation(JSONObject rawDoc, Instances instances) { + PhysicalLocationMap physMap = new PhysicalLocationMap(); + + String model = rawDoc.optString("model"); + String rootPid = rawDoc.optString("root.pid"); + List textLocations = new ArrayList<>(); + + if (rawDoc.has("physical_locations.facet")) { + JSONArray locations = rawDoc.optJSONArray("physical_locations.facet"); + for (int i = 0; i < locations.length(); i++) { + textLocations.add(locations.getString(i)); + } } else { - return new ArrayList<>(); + List list = KConfiguration.getInstance().getConfiguration().getList("cdk.inferring_physical_locations.model" , Arrays.asList("periodicalitem","monographunit")).stream().map(Object::toString).collect(Collectors.toList()); + if (list.contains(model)) { + textLocations = inferPhysicalLocationFromPredecessor(solrAccess, rootPid); + } } + return textLocations.stream().map(sigla-> { + return physMap.findBySigla(sigla); + }).map(acronym-> { + if (acronym != null) { + if (instances.isEnabledInstance(acronym)) { + return acronym; + } else { + return null; + } + } else { + return null; + } + }).filter(acronym -> acronym != null).collect(Collectors.toList()); } @Override @@ -227,7 +294,7 @@ public void filterValue(JSONObject rawDoc) { List dInsts = libraries.disabledInstances().stream().map(OneInstance::getName).collect(Collectors.toList()); if (rawDoc.has("cdk.collection")) { - List acronymsSortedByPhysicalLocation = acronymsSortedByPhysicalLocation(rawDoc.optJSONArray("physical_locations.facet"), libraries); + List acronymsSortedByPhysicalLocation = acronymsSortedByPhysicalLocation(rawDoc, libraries); JSONArray col = rawDoc.getJSONArray("cdk.collection"); JSONArray nCol = new JSONArray(); @@ -274,6 +341,31 @@ public String enhanceFacetsTerms() { } else return null; } + + + + private static List inferPhysicalLocationFromPredecessor(SolrAccess sa, String rootPid) { + try { + String encoded = URLEncoder.encode(String.format("root.pid:\"%s\"",rootPid), "UTF-8"); + String fl = URLEncoder.encode("pid physical_locations.facet own_pid_path","UTF-8"); + String solrResponseJson = sa.requestWithSelectReturningString( String.format("q=%s+AND+physical_locations.facet:*&fl=%s&rows=4000",encoded,fl), "json"); + JSONObject response = new JSONObject(solrResponseJson); + + List physicalLocations = new ArrayList<>(); + JSONArray jsonArray = response.getJSONObject("response").getJSONArray("docs"); + if (jsonArray.length() > 0 ) { + JSONObject oneDoc = jsonArray.getJSONObject(0); + JSONArray physJsonArray = oneDoc.getJSONArray("physical_locations.facet"); + for (int i = 0; i < physJsonArray.length(); i++) { + physicalLocations.add(physJsonArray.getString(i)); + } + } + return physicalLocations; + } catch (JSONException | IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + return new ArrayList<>(); + } + } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/PhysicalLocationMap.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/PhysicalLocationMap.java index f046d39cb2..11bcfd3665 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/PhysicalLocationMap.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/PhysicalLocationMap.java @@ -1,5 +1,6 @@ package cz.incad.kramerius.rest.apiNew.client.v60.libs; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.List; @@ -78,5 +79,12 @@ public String findBySigla(String sigla) { } return null; } - + + public List getAssocations(String acronym) { + if (MAPPING.containsKey(acronym)) { + return MAPPING.get(acronym); + } else { + return new ArrayList<>(); + } + } } diff --git a/rest/src/test/resources/cz/incad/kramerius/rest/apiNew/client/v60/filter/filter_simple_physicalfacets.json b/rest/src/test/resources/cz/incad/kramerius/rest/apiNew/client/v60/filter/filter_simple_physicalfacets.json new file mode 100644 index 0000000000..872893a970 --- /dev/null +++ b/rest/src/test/resources/cz/incad/kramerius/rest/apiNew/client/v60/filter/filter_simple_physicalfacets.json @@ -0,0 +1,85 @@ +{ + "response": { + "docs": [ + { + "titles.search": [ + "EHRENFRIDI HAGENDORNII ... HISTORIAE MEDICO-PHYSICAE, CENTURIIS TRIBUS comprehensae." + ], + "root.pid": "uuid:3d525aa8-b93a-4ba5-b1e7-8373800878d6", + "accessibility": "public", + "date_range_start.year": 1690, + "pid": "uuid:3d525aa8-b93a-4ba5-b1e7-8373800878d6", + "physical_locations.facet": [ + "ABA007" + ], + "foster_parents.pids": [ + "uuid:3d525aa8-b93a-4ba5-b1e7-8373800878d6" + ], + "rels_ext_index.sort": 0, + "id_barcode": [ + "26005725210" + ], + "pid_paths": [ + "uuid:3d525aa8-b93a-4ba5-b1e7-8373800878d6" + ], + "modified": "2023-11-08T11:28:18.911Z", + "model": "monograph", + "keywords.facet": [ + "Staré tisky", + "lékařství" + ], + "date.min": "1690-01-01T00:00:00.001Z", + "title.search": "EHRENFRIDI HAGENDORNII ... HISTORIAE MEDICO-PHYSICAE, CENTURIIS TRIBUS comprehensae.", + "root.title.sort": "EHRENFRIDI HAGENDORNII HISTORIAE MEDICO PHYSICAE CENTURIIS TRIBUS COMPREHENSAE ", + "cdk.collection": [ + "nkp", + "knav", + "mzk" + ], + "root.title": "EHRENFRIDI HAGENDORNII ... HISTORIAE MEDICO-PHYSICAE, CENTURIIS TRIBUS comprehensae.", + "indexed": "2023-11-08T11:28:18.911Z", + "level": 0, + "created": "2023-11-01T14:13:54.814Z", + "own_model_path": "monograph", + "languages.facet": [ + "lat" + ], + "cdk.leader": "knav", + "title.sort": "EHRENFRIDI HAGENDORNII HISTORIAE MEDICO PHYSICAE CENTURIIS TRIBUS COMPREHENSAE ", + "date.max": "1690-01-01T00:00:00.001Z", + "date_range_end.year": 1690, + "ddt": [ + "" + ], + "_version_": 1782028536889999400, + "shelf_locators": [ + "TH 316/adl.2" + ], + "date.str": "1690", + "compositeId": "uuid:3d525aa8-b93a-4ba5-b1e7-8373800878d6!uuid:3d525aa8-b93a-4ba5-b1e7-8373800878d6", + "keywords.search": [ + "Staré tisky", + "lékařství" + ], + "own_pid_path": "uuid:3d525aa8-b93a-4ba5-b1e7-8373800878d6", + "own_parent.pid": "uuid:3d525aa8-b93a-4ba5-b1e7-8373800878d6" + } + ], + "numFound": 1, + "start": 0, + "maxScore": 7.6994467, + "numFoundExact": true + }, + "responseHeader": { + "zkConnected": true, + "QTime": 3, + "params": { + "q": "pid:\"uuid:3d525aa8-b93a-4ba5-b1e7-8373800878d6\"", + "hl.fragsize": "20", + "fq": "", + "rows": "1", + "wt": "json" + }, + "status": 0 + } +} \ No newline at end of file diff --git a/rest/src/test/resources/cz/incad/kramerius/rest/apiNew/client/v60/filter/filter_simple_physicalfacets.xml b/rest/src/test/resources/cz/incad/kramerius/rest/apiNew/client/v60/filter/filter_simple_physicalfacets.xml new file mode 100644 index 0000000000..edeed85e6a --- /dev/null +++ b/rest/src/test/resources/cz/incad/kramerius/rest/apiNew/client/v60/filter/filter_simple_physicalfacets.xml @@ -0,0 +1,87 @@ + + + + true + 0 + 58 + + pid:"uuid:87f93b08-e35d-410c-a794-b8fe52cc6400" + 20 + + xml + xml + + + + + + private + + Gottlieb, František, 1903-1974 + + + Gottlieb, František + + + Gottlieb, František + + 2022-10-10T15:28:13.756Z + 1924-12-31T23:59:59.999Z + 1924-01-01T00:00:00.001Z + 1924 + 1924 + 1924 + image/jpeg + true + + 87f93b08-e35d-410c-a794-b8fe52cc6400 + + 2022-10-12T22:56:54.205Z + 13 + + cze + + 1 + page + 2022-10-13T00:29:12.444Z + monograph/page + monograph + uuid:e8686650-d81d-11ec-a96b-005056827e52 + Cesta do Kanaán: verše + uuid:e8686650-d81d-11ec-a96b-005056827e52/uuid:87f93b08-e35d-410c-a794-b8fe52cc6400 + + 9 + [6] + left + NormalPage + uuid:87f93b08-e35d-410c-a794-b8fe52cc6400 + + uuid:e8686650-d81d-11ec-a96b-005056827e52/uuid:87f93b08-e35d-410c-a794-b8fe52cc6400 + + + 8 + monograph + uuid:e8686650-d81d-11ec-a96b-005056827e52 + Cesta do Kanaán: verše + CESTA DO KANAA|N VERS|E + [6] + 6 + uuid:e8686650-d81d-11ec-a96b-005056827e52!uuid:87f93b08-e35d-410c-a794-b8fe52cc6400 + + + mzk + nkp + knav + + + + ABA001 + ABB060 + + + mzk + 1747194390056534016 + + + \ No newline at end of file From 7c318fd74199a3cfd36f17a02609d811520600f3 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 17 Nov 2023 08:18:30 +0100 Subject: [PATCH 093/235] Issue https://github.com/ceskaexpedice/ceska-digitalni-knihovna/issues/49 --- .../apiNew/client/v60/ClientUserResource.java | 59 ++++++++++--------- .../client/v60/filter/DefaultFilter.java | 17 +++--- .../client/v60/filter/DefaultFilterTest.java | 2 +- .../searchIndex/indexer/SolrIndexAccess.java | 10 ++++ .../cz/incad/kramerius/processes/res/lp.st | 13 ++++ 5 files changed, 64 insertions(+), 37 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientUserResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientUserResource.java index 4e7316776f..97d9bccf4f 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientUserResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientUserResource.java @@ -162,36 +162,37 @@ public Response info(@QueryParam("sessionAttributes") String sessionAttributes) } private List findLabels(User user) { - List licenses = new ArrayList<>(); - - Client client = this.clientProvider.get(); - String remoteAddress = IPAddressUtils.getRemoteAddress(this.provider.get(), KConfiguration.getInstance().getConfiguration()); - - List userAggegations = new ArrayList<>(); - List eInstances = this.instances.enabledInstances(); - eInstances.forEach(eI -> { - if (eI.hasFullAccess()) { - userAggegations.add(eI); - } - }); - - for (OneInstance oneInstance : userAggegations) { - try { - ProxyUserHandler proxyUserHandler = oneInstance.createProxyUserHandler(user, client, solrAccess, oneInstance.getName(), remoteAddress); - Pair> retval = proxyUserHandler.user(); - if (retval != null) { - licenses.addAll(retval.getValue()); - Map sessionAttributes = retval.getKey().getSessionAttributes(); - sessionAttributes.keySet().forEach(key-> { - user.addSessionAttribute(oneInstance.getName()+"_"+key, sessionAttributes.get(key)); - }); - - } - } catch (ProxyHandlerException e) { - LOGGER.log(Level.SEVERE,e.getMessage()); - } - } + boolean detectLabels = KConfiguration.getInstance().getConfiguration().getBoolean("cdk.infer.licenses", true); + if (detectLabels) { + Client client = this.clientProvider.get(); + String remoteAddress = IPAddressUtils.getRemoteAddress(this.provider.get(), KConfiguration.getInstance().getConfiguration()); + + List userAggegations = new ArrayList<>(); + List eInstances = this.instances.enabledInstances(); + eInstances.forEach(eI -> { + if (eI.hasFullAccess()) { + userAggegations.add(eI); + } + }); + + for (OneInstance oneInstance : userAggegations) { + try { + ProxyUserHandler proxyUserHandler = oneInstance.createProxyUserHandler(user, client, solrAccess, oneInstance.getName(), remoteAddress); + Pair> retval = proxyUserHandler.user(); + if (retval != null) { + licenses.addAll(retval.getValue()); + Map sessionAttributes = retval.getKey().getSessionAttributes(); + sessionAttributes.keySet().forEach(key-> { + user.addSessionAttribute(oneInstance.getName()+"_"+key, sessionAttributes.get(key)); + }); + + } + } catch (ProxyHandlerException e) { + LOGGER.log(Level.SEVERE,e.getMessage()); + } + } + } return licenses; } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java index f33017a4b7..c436cd6044 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java @@ -302,18 +302,21 @@ public void filterValue(JSONObject rawDoc) { String leader = null; rawDoc.put("cdk.collection.sorted", true); - List collectionFromIndex = new ArrayList<>(); - for (int i = 0; i < col.length(); i++) { collectionFromIndex.add(col.getString(i)); } + List processCollectionFromIndex = new ArrayList<>(); + List allCollections = new ArrayList<>(); + for (int i = 0; i < col.length(); i++) { processCollectionFromIndex.add(col.getString(i)); allCollections.add(col.getString(i));} while(!acronymsSortedByPhysicalLocation.isEmpty()) { String acronym = acronymsSortedByPhysicalLocation.remove(0); - if (leader == null) leader = acronym; - nCol.put(acronym); - collectionFromIndex.remove(acronym); + if (allCollections.contains(acronym)) { + if (leader == null) leader = acronym; + nCol.put(acronym); + processCollectionFromIndex.remove(acronym); + } } - for (int i = 0; i < collectionFromIndex.size(); i++) { - if (!dInsts.contains(collectionFromIndex.get(i))) nCol.put(collectionFromIndex.get(i)); + for (int i = 0; i < processCollectionFromIndex.size(); i++) { + if (!dInsts.contains(processCollectionFromIndex.get(i))) nCol.put(processCollectionFromIndex.get(i)); } if (leader != null) { diff --git a/rest/src/test/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilterTest.java b/rest/src/test/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilterTest.java index c090416e8f..7f6524ca72 100644 --- a/rest/src/test/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilterTest.java +++ b/rest/src/test/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilterTest.java @@ -376,7 +376,7 @@ public void testValueDocWithPhysicalLocationJSON() throws IOException { EasyMock.replay(inst); - ProxyFilter pf = new DefaultFilter(inst); + ProxyFilter pf = new DefaultFilter(inst,null); InputStream stream = DefaultFilterTest.class.getResourceAsStream("filter_simple_physicalfacets.json"); String json = IOUtils.toString(stream,"UTF-8"); diff --git a/search-index/src/main/java/cz/kramerius/searchIndex/indexer/SolrIndexAccess.java b/search-index/src/main/java/cz/kramerius/searchIndex/indexer/SolrIndexAccess.java index 9fb8d8cdf6..811d920afa 100644 --- a/search-index/src/main/java/cz/kramerius/searchIndex/indexer/SolrIndexAccess.java +++ b/search-index/src/main/java/cz/kramerius/searchIndex/indexer/SolrIndexAccess.java @@ -152,6 +152,16 @@ public UpdateResponse deleteByIds(List ids) throws IOException, SolrServ return null; } +// public UpdateResponse deleteByRootPid(String rootPid) throws SolrServerException, IOException { +// //System.out.println("deleting all"); +// UpdateResponse deleteResponse = solrClient.deleteByQuery(collection, "root.pid:\""+rootPid+"\""); +// //System.out.println("delete response: " + deleteResponse); +// UpdateResponse commitResponse = solrClient.commit(collection); +// //System.out.println("commit response: " + commitResponse); +// return null; +// +// } + public UpdateResponse deleteAll() throws IOException, SolrServerException { //System.out.println("deleting all"); UpdateResponse deleteResponse = solrClient.deleteByQuery(collection, "*"); diff --git a/shared/common/src/main/java/cz/incad/kramerius/processes/res/lp.st b/shared/common/src/main/java/cz/incad/kramerius/processes/res/lp.st index d869bed70e..9fc8d11c31 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/processes/res/lp.st +++ b/shared/common/src/main/java/cz/incad/kramerius/processes/res/lp.st @@ -737,4 +737,17 @@ + + + reharvest + CDK / Reharvest pid + + cz.incad.kramerius.ReharvestPids + + lrOut + + lrErr + -Xmx4096m -Xms256m -Dfile.encoding=UTF-8 + + From d4df795ad6debd0d2cdb479dd8cecbf8854dace2 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 17 Nov 2023 10:41:22 +0100 Subject: [PATCH 094/235] Issue https://github.com/ceskaexpedice/ceska-digitalni-knihovna/issues/49 --- .../v10/proxy/ConnectedInfoResource.java | 14 +++++- .../client/v60/libs/PhysicalLocationMap.java | 45 +++++++++++++++++++ 2 files changed, 57 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java index 4f7c6ffa98..11d6a6daf0 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java @@ -88,8 +88,18 @@ public Response assocations(@PathParam("library") String library) { PhysicalLocationMap locationMap = new PhysicalLocationMap(); List assocations = locationMap.getAssocations(library); JSONArray restArr = new JSONArray(); - assocations.stream().forEach(restArr::put); - return Response.ok(restArr).build(); + + assocations.stream().forEach( sigla-> { + String desc = locationMap.getDescription(sigla); + JSONObject obj = new JSONObject(); + obj.put("sigla", sigla); + if (desc == null) { + desc = ""; + } + obj.put("description", desc); + restArr.put(obj); + }); + return Response.ok(restArr.toString()).build(); } @PUT diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/PhysicalLocationMap.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/PhysicalLocationMap.java index 11bcfd3665..ca24a5f46f 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/PhysicalLocationMap.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/PhysicalLocationMap.java @@ -9,7 +9,9 @@ public class PhysicalLocationMap { private static final Map> MAPPING = new HashMap<>(); + private static final Map DESCRIPTIONS = new HashMap<>(); + static { MAPPING.put("nkp", Arrays.asList( "ABA001", @@ -66,6 +68,46 @@ public class PhysicalLocationMap { MAPPING.put("vkol", Arrays.asList( "OLA001")); + + + DESCRIPTIONS.put("ABA000", "Národni knihovna ČR"); + DESCRIPTIONS.put("ABA001", "Národni knihovna ČR"); + DESCRIPTIONS.put("ABA004", "Národni knihovna ČR - Slovanská knihovna"); + + DESCRIPTIONS.put("ABA007", "Knihovna AVČR"); + DESCRIPTIONS.put("ABB045", "Knihovna AVČR - Etnologický ústav AV ČR"); + DESCRIPTIONS.put("ABB030", "Knihovna AVČR - Orientální ústav AV ČR"); + DESCRIPTIONS.put("ABE459", "Knihovna AVČR - Královská kanonie premonstrátů"); + DESCRIPTIONS.put("ABB060", "Knihovna AVČR - Ústav pro českou literaturu AV ČR"); + + + DESCRIPTIONS.put("BOA001", "Moravská zemská knihovna v Brně"); + DESCRIPTIONS.put("KME450", "MZK - Muzeum umění Olomouc, Arcidiecézní muzeum Kroměříž"); + DESCRIPTIONS.put("BOE950", "MZK - Benediktinské opatství Rajhrad"); + DESCRIPTIONS.put("BOE801", "MZK - Muzeum Brněnska"); + DESCRIPTIONS.put("BVE301", "MZK - Regionální muzeum v Mikulově"); + DESCRIPTIONS.put("HOE802", "MZK - Masarykovo muzeum v Hodoníně"); + + + DESCRIPTIONS.put("HKA001", "Studijní a vědecká knihovna v Hradci Králové"); + DESCRIPTIONS.put("JCE301", "SVKHK - Regionální muzeum v Jičíně"); + DESCRIPTIONS.put("RKE801", "SVKHK - Okresní muzeum Orlických hor"); + DESCRIPTIONS.put("NAG001", "SVKHK - Městská knihovna Náchod"); + DESCRIPTIONS.put("HKE302", "SVKHK - Muzeum východních Čech v Hradci Králové"); + DESCRIPTIONS.put("NAE802", "SVKHK - Městské muzeum v Jaroměři"); + DESCRIPTIONS.put("HKG001", "SVKHK - Knihovna města Hradce Králové"); + DESCRIPTIONS.put("TUE301", "SVKHK - Muzeum Podkrkonoší v Trutnově"); + DESCRIPTIONS.put("JCG001", "SVKHK - Knihovna Václava Čtvrtka v Jičíně"); + + DESCRIPTIONS.put("ABA009", "Knihovna Antonína Švehly"); + DESCRIPTIONS.put("PAG001", "Krajská knihovna v Pardubicích"); + DESCRIPTIONS.put("ULG001", "Knihovna ústeckého kraje"); + DESCRIPTIONS.put("ABC135", "Národní filmový archiv"); + DESCRIPTIONS.put("ABG001", "Městská knihovna v Praze"); + DESCRIPTIONS.put("ABA010", "Národní muzeum"); + DESCRIPTIONS.put("CBA001", "Jihočeská vědecká knihovna"); + DESCRIPTIONS.put("OLA001", "Vědecká knihovna v Olomouci"); + } public PhysicalLocationMap() {} @@ -80,6 +122,9 @@ public String findBySigla(String sigla) { return null; } + public String getDescription(String sigla) { + return DESCRIPTIONS.get(sigla); + } public List getAssocations(String acronym) { if (MAPPING.containsKey(acronym)) { return MAPPING.get(acronym); From 8b81e2c39bf0c761926ca897b4b07ccf917ca080 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 17 Nov 2023 20:17:35 +0100 Subject: [PATCH 095/235] Issue https://github.com/ceskaexpedice/ceska-digitalni-knihovna/issues/49 --- .../kramerius/services/utils/SolrUtils.java | 4 +-- .../admin/v10/processes/ProcessResource.java | 27 ++++++++++++++++++- .../database/ProcessDatabaseUtils.java | 8 ++++-- .../cz/incad/kramerius/processes/res/lp.st | 2 ++ 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/SolrUtils.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/SolrUtils.java index b4a93d12c7..7141423228 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/SolrUtils.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/SolrUtils.java @@ -173,14 +173,14 @@ public boolean acceptElement(Element element) { public static Element executeQuery(Client client, String url, String query, String user, String pass) throws ParserConfigurationException, SAXException, IOException { String u = url+(url.endsWith("/") ? "" : "/")+ query; - LOGGER.info(String.format("[" + Thread.currentThread().getName() + "] url %s", u)); + LOGGER.fine(String.format("[" + Thread.currentThread().getName() + "] url %s", u)); WebResource r = client.resource(u); if (user != null && pass != null) { r.addFilter(new BasicAuthenticationClientFilter(user, pass)); } - LOGGER.info(String.format("[" + Thread.currentThread().getName() + "] processing %s", r.getURI().toString())); + LOGGER.fine(String.format("[" + Thread.currentThread().getName() + "] processing %s", r.getURI().toString())); String t = r.accept(MediaType.APPLICATION_XML).get(String.class); Document parseDocument = XMLUtils.parseDocument(new StringReader(t)); Stack stack = new Stack<>(); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/processes/ProcessResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/processes/ProcessResource.java index 57e7ab74c8..d857c911ec 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/processes/ProcessResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/processes/ProcessResource.java @@ -763,7 +763,7 @@ private JSONObject lrPRocessToJSONObject(LRProcess lrProcess) throws JSONExcepti jsonObject.put("userSurname", lrProcess.getSurname()); //empty return jsonObject; } - + //TODO: I18N private String buildInitialProcessName(String defId, List params) { try { switch (defId) { @@ -808,6 +808,10 @@ private String buildInitialProcessName(String defId, List params) { case "nkplogs": { return String.format("Generování NKP logů pro období %s - %s", params.get(0), params.get(1)); } + case "reharvest": { + return String.format("Reharvest objektu %s", params.get(0)); + } + case "delete_tree": { String pid = params.get(0); String title = params.get(1); @@ -1010,6 +1014,27 @@ private List paramsToList(String id, JSONObject params, Consumer pidlist = extractOptionalParamStringList(params, "pidlist", null); + File pidlistFile = extractOptionalParamFileContainedInADir(params, "pidlist_file", new File(KConfiguration.getInstance().getProperty("convert.directory"))); //TODO: specialni adresar pro pidlisty, ne convert.directory + String target; + if (pid != null) { + target = "pid:" + pid; + } else if (pidlist != null) { + target = "pidlist:" + pidlist.stream().collect(Collectors.joining(";")); + } else if (pidlistFile != null) { + target = "pidlist_file:" + pidlistFile.getAbsolutePath(); + } else { + throw new BadRequestException("target not specified, use one of following parameters: pid, pidlist, pidlist_file"); + } + + + List result = new ArrayList<>(); + result.add(target); + return result; + } case "delete_tree": { String pid = extractMandatoryParamWithValuePrefixed(params, "pid", "uuid:"); String title = extractOptionalParamString(params, "title", null); diff --git a/shared/common/src/main/java/cz/incad/kramerius/processes/database/ProcessDatabaseUtils.java b/shared/common/src/main/java/cz/incad/kramerius/processes/database/ProcessDatabaseUtils.java index 21c2d026d1..069b53ce47 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/processes/database/ProcessDatabaseUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/processes/database/ProcessDatabaseUtils.java @@ -109,7 +109,8 @@ public static int registerProcess(Connection con, LRProcess lp, String storedPar "auth_token," + //10 "ip_addr," + //11 "owner_id," + //12 - "owner_name" + //13 + "owner_name," + //13, + "name" + //14 ")" + "values " + " (" + @@ -126,7 +127,9 @@ public static int registerProcess(Connection con, LRProcess lp, String storedPar " ?," + //10 AUTH_TOKEN " ?," + //11 IP_ADDR " ?," + //12 OWNER_ID - " ?" + //13 OWNER_NAME + " ?," + //13 OWNER_NAME + " ?" + //14 NAME + " )"); insertStatement.setString(1, lp.getDefinitionId()); insertStatement.setString(2, lp.getUUID()); @@ -141,6 +144,7 @@ public static int registerProcess(Connection con, LRProcess lp, String storedPar insertStatement.setString(11, lp.getPlannedIPAddress()); insertStatement.setString(12, lp.getOwnerId()); insertStatement.setString(13, lp.getOwnerName()); + insertStatement.setString(14, lp.getProcessName()); insertStatement.executeUpdate(); //get new process_id diff --git a/shared/common/src/main/java/cz/incad/kramerius/processes/res/lp.st b/shared/common/src/main/java/cz/incad/kramerius/processes/res/lp.st index 9fc8d11c31..8565df2e86 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/processes/res/lp.st +++ b/shared/common/src/main/java/cz/incad/kramerius/processes/res/lp.st @@ -748,6 +748,8 @@ lrErr -Xmx4096m -Xms256m -Dfile.encoding=UTF-8 + a_admin_read + From e33ae61608365978ba59a47d8cda22ba57ec43f5 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 9 Jan 2024 10:42:54 +0100 Subject: [PATCH 096/235] Name of user; https://github.com/ceskaexpedice/ceska-digitalni-knihovna/issues/50 --- .../auth/thirdparty/keycloack/KeycloackUserSupport.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java b/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java index 27afe3b652..3a2238a69c 100644 --- a/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java +++ b/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java @@ -32,6 +32,8 @@ public class KeycloackUserSupport extends AbstractThirdPartyUsersSupport { + private static final String EDU_PERSON_UNIQUE_ID = "eduPersonUniqueId"; + public static final Logger LOGGER = Logger.getLogger(KeycloackUserSupport.class.getName()); public static final String KEYCLOACK_USER_PREFIX = "_keycloack_"; @@ -170,8 +172,9 @@ public String calculateUserName(HttpServletRequest request) { if (request.getUserPrincipal() instanceof KeycloakPrincipal) { AccessToken token = ((KeycloakPrincipal) request.getUserPrincipal()).getKeycloakSecurityContext() .getToken(); - if (token.getEmail() != null) { - return token.getEmail(); + // If the user is logged in via federation, the eduPersonUniqueId attribute is used; otherwise, the preferred_username attribute is used. + if (token.getOtherClaims().containsKey(EDU_PERSON_UNIQUE_ID)) { + return token.getOtherClaims().get(EDU_PERSON_UNIQUE_ID).toString(); } else { return token.getPreferredUsername(); } From 2649aab4cfbd59e3ed044206a1568ced2d9ad0b8 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 24 Jan 2024 20:20:54 +0100 Subject: [PATCH 097/235] Mime types --- .../sliced/nkp/nkp_sliced_12.xml | 4 +- .../sliced/nkp/nkp_sliced_13.xml | 93 +++++ .../sliced/nkp/nkp_sliced_14.xml | 93 +++++ .../sliced/nkp/nkp_sliced_15.xml | 93 +++++ .../sliced/nkp/nkp_sliced_16.xml | 93 +++++ .../sliced/nkp/nkp_sliced_17.xml | 93 +++++ .../sliced/nkp/nkp_sliced_19.xml | 93 +++++ .../sliced/nkp/nkp_sliced_2.xml | 6 +- .../sliced/nkp/nkp_sliced_20.xml | 93 +++++ .../sliced/nkp/nkp_sliced_21.xml | 93 +++++ .../sliced/nkp/nkp_sliced_22.xml | 93 +++++ .../sliced/nkp/nkp_sliced_23.xml | 93 +++++ .../sliced/nkp/nkp_sliced_24.xml | 93 +++++ .../sliced/nkp/nkp_sliced_25.xml | 93 +++++ .../sliced/nkp/nkp_sliced_26.xml | 93 +++++ .../sliced/nkp/nkp_sliced_3.xml | 2 +- processes/cdkreharvest/build.gradle | 16 + .../cz/incad/kramerius/ReharvestPids.java | 328 ++++++++++++++++++ .../v60/redirection/ProxyHandlerSupport.java | 9 +- settings.gradle | 2 + 20 files changed, 1567 insertions(+), 9 deletions(-) create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_13.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_14.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_15.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_16.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_17.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_19.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_20.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_21.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_22.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_23.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_24.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_25.xml create mode 100644 processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_26.xml create mode 100644 processes/cdkreharvest/build.gradle create mode 100644 processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_12.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_12.xml index 68068838b3..41206efe82 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_12.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_12.xml @@ -17,8 +17,8 @@ 5000 CURSOR - cdkmzk - CCYeZPSe + $iteration.user$ + $iteration.pass$ modified_date diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_13.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_13.xml new file mode 100644 index 0000000000..2c44ef3aff --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_13.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2017-06-01T00:00:00.000Z TO 2018-01-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_14.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_14.xml new file mode 100644 index 0000000000..e4ee296a0a --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_14.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2018-01-01T00:00:00.000Z TO 2018-06-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_15.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_15.xml new file mode 100644 index 0000000000..ff65003b9c --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_15.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2018-06-01T00:00:00.000Z TO 2019-01-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_16.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_16.xml new file mode 100644 index 0000000000..4c0093a5e6 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_16.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2019-01-01T00:00:00.000Z TO 2019-06-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_17.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_17.xml new file mode 100644 index 0000000000..966abb72bd --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_17.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2019-06-01T00:00:00.000Z TO 2020-01-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_19.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_19.xml new file mode 100644 index 0000000000..550568aaca --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_19.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2020-01-01T00:00:00.000Z TO 2020-06-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_2.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_2.xml index 1f83543776..197620a4f0 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_2.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_2.xml @@ -83,9 +83,9 @@ PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model - - nkp - + + nkp + diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_20.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_20.xml new file mode 100644 index 0000000000..8ee7237f32 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_20.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2020-06-01T00:00:00.000Z TO 2021-01-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_21.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_21.xml new file mode 100644 index 0000000000..7120fe75bb --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_21.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2021-01-01T00:00:00.000Z TO 2021-06-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_22.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_22.xml new file mode 100644 index 0000000000..b58e3ab4f6 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_22.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2021-06-01T00:00:00.000Z TO 2022-01-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_23.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_23.xml new file mode 100644 index 0000000000..cd5368fb07 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_23.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2022-01-01T00:00:00.000Z TO 2022-06-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_24.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_24.xml new file mode 100644 index 0000000000..bc6eadd01c --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_24.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2022-06-01T00:00:00.000Z TO 2023-01-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_25.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_25.xml new file mode 100644 index 0000000000..610699ff64 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_25.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2023-01-01T00:00:00.000Z TO 2024-01-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_26.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_26.xml new file mode 100644 index 0000000000..15b4233906 --- /dev/null +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_26.xml @@ -0,0 +1,93 @@ + + + nkp + nkp-full + full + + $iteration.workingtime$ + + + + + $iteration.url$ + modified_date:[2023-11-01T00:00:00.000Z TO 2024-02-01T00:00:00.000Z] + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + + modified_date + + + select + pid + 4000 + CURSOR + + + + 1 + + + + + + $iteration.url$ + select + 88 + + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + $destination.url$ + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + nkp + nkp + + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + nkp + + + + + + \ No newline at end of file diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_3.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_3.xml index 4300d8703f..4f2f526e86 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_3.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/sliced/nkp/nkp_sliced_3.xml @@ -83,7 +83,7 @@ PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model - + nkp diff --git a/processes/cdkreharvest/build.gradle b/processes/cdkreharvest/build.gradle new file mode 100644 index 0000000000..0c319b3ed2 --- /dev/null +++ b/processes/cdkreharvest/build.gradle @@ -0,0 +1,16 @@ +apply plugin: 'java' + + +dependencies { + api project(':shared:common') + api project(':processes:cdkprocesses') + + implementation project(':processes:solr-migration') + + implementation project(':search-index') + + testImplementation 'org.xmlunit:xmlunit-core:2.8.4' + testImplementation group: 'org.xmlunit', name: 'xmlunit-matchers', version: '2.8.4' + testImplementation group: 'org.xmlunit', name: 'xmlunit-assertj', version: '2.8.4' + +} diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java new file mode 100644 index 0000000000..87f67fe7e6 --- /dev/null +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java @@ -0,0 +1,328 @@ +package cz.incad.kramerius; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.net.URLEncoder; +import java.nio.charset.Charset; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; + +import org.antlr.stringtemplate.StringTemplate; +import org.antlr.stringtemplate.language.DefaultTemplateLexer; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.solr.client.solrj.SolrServerException; +import org.w3c.dom.DOMException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import com.google.common.io.Files; +import com.google.inject.Guice; +import com.google.inject.Injector; +import com.google.inject.Key; +import com.hazelcast.config.KubernetesConfig; + +import cz.incad.kramerius.fedora.RepoModule; +import cz.incad.kramerius.fedora.om.RepositoryException; +import cz.incad.kramerius.impl.SolrAccessImplNewIndex; +import cz.incad.kramerius.resourceindex.ResourceIndexException; +import cz.incad.kramerius.resourceindex.ResourceIndexModule; +import cz.incad.kramerius.service.MigrateSolrIndexException; +import cz.incad.kramerius.solr.SolrModule; +import cz.incad.kramerius.statistics.NullStatisticsModule; +import cz.incad.kramerius.utils.StringUtils; +import cz.incad.kramerius.utils.XMLUtils; +import cz.incad.kramerius.utils.conf.KConfiguration; +//import cz.incad.kramerius.utils.solr.SolrUtils; +import cz.kramerius.searchIndex.indexer.SolrConfig; +import cz.kramerius.searchIndex.indexer.SolrIndexAccess; +import cz.incad.kramerius.services.KubernetesProcessImpl; +import cz.incad.kramerius.services.ParallelProcessImpl; +import cz.incad.kramerius.services.utils.ResultsUtils; +import cz.incad.kramerius.services.utils.SolrUtils; + + +public class ReharvestPids { + + public static final Logger LOGGER = Logger.getLogger(ReharvestPids.class.getName()); + + + /** + * args[0] - action (ADD/REMOVE), from lp.st process/parameters + * args[1] - authToken + * args[2] - target (pid:uuid:123, or pidlist:uuid:123;uuid:345;uuid:789, or pidlist_file:/home/kramerius/.kramerius/import-dnnt/grafiky.txt + * In case of pidlist pids must be separated with ';'. Convenient separator ',' won't work due to way how params are stored in database and transferred to process. + *

+ * args[3] - licence ('dnnt', 'dnnto', 'public_domain', etc.) + * @throws TransformerException + * @throws ParserConfigurationException + * @throws MigrateSolrIndexException + */ + public static void main(String[] args) throws IOException, SolrServerException, RepositoryException, ResourceIndexException, TransformerException, ParserConfigurationException, MigrateSolrIndexException { + if (args.length < 2) { + throw new RuntimeException("Not enough arguments."); + } + LOGGER.info("Parameters "+Arrays.asList(args)); + int argsIndex = 0; +// //params from lp.st + String authToken = args[argsIndex++]; + String target = args[argsIndex++]; //auth token always second, but still suboptimal solution, best would be if it was outside the scope of this as if ProcessHelper.scheduleProcess() similarly to changing name (ProcessStarter) + List extractPids = extractPids(target); + for (String pid : extractPids) { reharvestPID(pid); } + } + + private static void reharvestPID(String pid) + throws IOException, TransformerException, MigrateSolrIndexException, ParserConfigurationException { + Injector injector = Guice.createInjector(new SolrModule(), new ResourceIndexModule(), new RepoModule(), new NullStatisticsModule(), new ResourceIndexModule()); + + SolrAccess searchIndex = injector.getInstance(Key.get(SolrAccessImplNewIndex.class)); //FIXME: hardcoded implementation + Document document = searchIndex.getSolrDataByPid(pid); + int nf = numFound(document); + if (nf > 0) { + + String msg = String.format("reharvesting pid %s ", pid); + LOGGER.info(msg); + String leader = leader(document); + List collections = collections(document); + + ParallelProcessImpl parallelProcess = new ParallelProcessImpl(); + Document deleteByQuery = deleteRootPid(searchIndex, pid); + + String s = SolrUtils.sendToDest(getDestinationUpdateUrl(), parallelProcess.getClient(), deleteByQuery); + + List harvestFiles = new ArrayList<>(); + + // leader must be first + collections.remove(leader); + collections.add(0, leader); + + for (String ac : collections) { + try { + String api = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + ac + ".api"); + + Pair iterationUrl = iterationUrl(ac); + + Map iteration = new HashMap<>(); + iteration.put("url", iterationUrl.getKey()); + iteration.put("dl", ac); + iteration.put("fquery", fq(api, pid)); + + + if (iterationUrl.getRight()) { + String username = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + ac + ".username"); + String password = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + ac + ".pswd"); + + iteration.put("user", username); + iteration.put("pass", password); + } + + // destination + Map destination = new HashMap<>(); + destination.put("url", KConfiguration.getInstance().getSolrSearchHost()); + + boolean publicEndpoint = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + ac + ".public") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + ac + ".public") : false; + + String configuration = renderTemplate(publicEndpoint, api, iteration, destination); + File tmpFile = File.createTempFile(String.format("%s", ac), "reharvest"); + + Files.write(configuration.getBytes("UTF-8"), tmpFile); + harvestFiles.add(tmpFile); + + } catch (IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } + } + + for (File harvestFile : harvestFiles) { + try { + ParallelProcessImpl reharvest = new ParallelProcessImpl(); + String config = org.apache.commons.io.IOUtils.toString(new FileInputStream(harvestFile), "UTF-8"); + reharvest.migrate(harvestFile); + + } catch (IOException | MigrateSolrIndexException | IllegalAccessException | InstantiationException | ClassNotFoundException | NoSuchMethodException | ParserConfigurationException | SAXException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } + } + } else { + String msg = String.format("pid %s not found", pid); + LOGGER.warning( msg); + } + } + + private static String fq(String api, String pid) { + switch(api) { + case "v7": return "root.pid:\""+pid+"\""; + case "v5": return "root_pid:\""+pid+"\""; + } + return "root_pid:\""+pid+"\""; + } + + private static String renderTemplate(boolean pbl, String api, Map iteration, Map destination) throws IOException { + String templatePath = String.format("reharvest_%s.xml", api); + if (pbl) { + templatePath = String.format("reharvest_public_%s.xml", api); + } + + InputStream stream = ReharvestPids.class.getResourceAsStream(templatePath); + + StringTemplate template = new StringTemplate( + org.apache.commons.io.IOUtils.toString(stream, "UTF-8"), DefaultTemplateLexer.class); + + template.setAttribute("iteration", iteration); + template.setAttribute("check", new HashMap<>()); + template.setAttribute("destination", destination); + template.setAttribute("timestamp", new HashMap<>()); + + String configuration = template.toString(); + return configuration; + } + + public static String getDestinationUpdateUrl() { + String searchHost = KConfiguration.getInstance().getSolrSearchHost(); + return searchHost +(searchHost.endsWith("/") ? "" : "/")+"update?commit=true"; + } + + public static Pair iterationUrl(String acronym) { + String baseurl = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + acronym + ".baseurl"); + String api = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + acronym + ".api"); + boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + acronym + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + acronym + ".licenses") : false; + String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + acronym + ".forwardurl"); + if (channelAccess) { + if (StringUtils.isAnyString(api) && api.toLowerCase().equals("v5")) { + String retval = channel + (channel.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/sync/solr"; + return Pair.of(retval, false); + } else { + String retval = channel + (channel.endsWith("/") ? "" : "/") + "api/v7.0/cdk/forward/sync/solr"; + return Pair.of(retval, false); + } + } else { + boolean publicEndpoint = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + acronym + ".public") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + acronym + ".public") : false; + if (publicEndpoint) { + String retval = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0"; + return Pair.of(retval, false); + } else { + String retval = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v4.6/cdk/solr"; + return Pair.of(retval, true); + + } + } + } + + + private static Document deleteRootPid(SolrAccess solrAccess, String pid) throws ParserConfigurationException, DOMException, IOException { + Document deleteByQuery = XMLUtils.crateDocument("delete"); + Element queryElement = deleteByQuery.createElement("query"); + queryElement.setTextContent(query(solrAccess, pid)); + deleteByQuery.getDocumentElement().appendChild(queryElement); + return deleteByQuery; + } + + private static String query(SolrAccess sAccess, String pid) throws IOException { + String query = String.format("root.pid:\"%s\"",pid); + if (query.startsWith("root.pid:\"uuid")) { + int threshold = KConfiguration.getInstance().getConfiguration().getInt("cdk.reharvest.items.threshold",80000); + + Document doc = sAccess.requestWithSelectReturningXml("q="+URLEncoder.encode( query, "UTF-8")); + int numFound = numFound(doc); + if (numFound > threshold) { + throw new IllegalStateException("Too many items to reharvest"); + } + return query; + } else throw new IllegalStateException("Pid must start with uuid !"); + } + +// private static Document deleteOwnPidPathSubtree(String pidPath) throws ParserConfigurationException { +// Document deleteByQuery = XMLUtils.crateDocument("delete"); +// Element queryElement = deleteByQuery.createElement("query"); +// queryElement.setTextContent(String.format("own_pid_path:%s/*", pidPath.replace(":", "\\:"))); +// deleteByQuery.getDocumentElement().appendChild(queryElement); +// return deleteByQuery; +// } + + + private static List collections(Document document) { + Element cdkCollections = XMLUtils.findElement(document.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + return name != null && name.equals("cdk.collection"); + } + }); + return cdkCollections != null ? XMLUtils.getElements(cdkCollections).stream().map(Element::getTextContent).map(String::trim).collect(Collectors.toList()) : new ArrayList<>(); + } + + private static String leader(Document document) { + Element cdkLeader = XMLUtils.findElement(document.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + return name != null && name.equals("cdk.leader"); + } + }); + return cdkLeader != null ? cdkLeader.getTextContent().trim() : null; + } + + public static int numFound(Document document) { + Element result = XMLUtils.findElement(document.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String nodeName = element.getNodeName(); + return nodeName.equals("result"); + } + }); + + String nf = result.getAttribute("numFound"); + return Integer.parseInt(nf); + } + + +// private static String ownPidPath(Document document) { +// Element cdkLeader = XMLUtils.findElement(document.getDocumentElement(), new XMLUtils.ElementsFilter() { +// @Override +// public boolean acceptElement(Element element) { +// String name = element.getAttribute("name"); +// return name != null && name.equals("own_pid_path"); +// } +// }); +// return cdkLeader != null ? cdkLeader.getTextContent().trim() : null; +// } + + private static List extractPids(String target) { + if (target.startsWith("pid:")) { + String pid = target.substring("pid:".length()); + List result = new ArrayList<>(); + result.add(pid); + return result; + } else if (target.startsWith("pidlist:")) { + List pids = Arrays.stream(target.substring("pidlist:".length()).split(";")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList()); + return pids; + } else if (target.startsWith("pidlist_file:")) { + String filePath = target.substring("pidlist_file:".length()); + File file = new File(filePath); + if (file.exists()) { + try { + return IOUtils.readLines(new FileInputStream(file), Charset.forName("UTF-8")); + } catch (IOException e) { + throw new RuntimeException("IOException " + e.getMessage()); + } + } else { + throw new RuntimeException("file " + file.getAbsolutePath() + " doesnt exist "); + } + } else { + throw new RuntimeException("invalid target " + target); + } + } +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 4c17270bf1..a39310dd67 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -100,7 +100,9 @@ public Response buildForwardResponseGET(String url, String mimetype) throws Prox ClientResponse response = b.get(ClientResponse.class); if (response.getStatus() == 200) { - InputStream is = response.getEntityInputStream(); + String responseMimeType = response.getType().toString(); + + InputStream is = response.getEntityInputStream(); MultivaluedMap headers = response.getHeaders(); StreamingOutput stream = new StreamingOutput() { @@ -114,8 +116,9 @@ public void write(OutputStream output) throws IOException, WebApplicationExcepti }; ResponseBuilder respEntity = null; if (mimetype != null) { - //MimeType type = new MimeType(mimetype); respEntity = Response.status(200).entity(stream).type(mimetype); + } else if (responseMimeType != null) { + respEntity = Response.status(200).entity(stream).type(responseMimeType); } else { respEntity = Response.status(200).entity(stream); } @@ -188,7 +191,7 @@ protected WebResource.Builder buidFowrardResponse(String url) { if (this.remoteAddr != null) { header = header + "|" + "header_ip_address=" + this.remoteAddr; } - + //TODO: Source if (StringUtils.isAnyString(prefixHeaders)) { header = prefixHeaders+header; } diff --git a/settings.gradle b/settings.gradle index 446463d203..fa7b184bc4 100644 --- a/settings.gradle +++ b/settings.gradle @@ -19,6 +19,8 @@ include 'processes:import-jaxb-mets' include 'processes:import-jaxb-srwdc' include 'processes:import-mets' include 'processes:cdkprocesses' +include 'processes:cdkreharvest' + include 'processes:solr-migration' //dependencies From 0b8875148027afbcc92ad767ce9fa16a5b699392 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 24 Jan 2024 22:12:39 +0100 Subject: [PATCH 098/235] OAI Support --- .../rest/api/guice/ApiServletModule.java | 4 + .../apiNew/client/v60/ClientApiResource.java | 12 + .../rest/oai/DeletedRecordsDetection.java | 21 + .../incad/kramerius/rest/oai/ErrorCode.java | 38 ++ .../kramerius/rest/oai/MetadataExport.java | 270 +++++++++++ .../incad/kramerius/rest/oai/OAIEndpoint.java | 107 +++++ .../incad/kramerius/rest/oai/OAIRecord.java | 118 +++++ .../incad/kramerius/rest/oai/OAIResults.java | 53 +++ .../cz/incad/kramerius/rest/oai/OAISet.java | 181 ++++++++ .../cz/incad/kramerius/rest/oai/OAISets.java | 153 ++++++ .../cz/incad/kramerius/rest/oai/OAITools.java | 133 ++++++ .../cz/incad/kramerius/rest/oai/OAIVerb.java | 439 ++++++++++++++++++ .../rest/oai/exceptions/OAIException.java | 122 +++++ 13 files changed, 1651 insertions(+) create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/oai/DeletedRecordsDetection.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/oai/ErrorCode.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/oai/OAIEndpoint.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/oai/OAIResults.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/oai/OAISets.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/oai/OAITools.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/oai/exceptions/OAIException.java diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java index cb7133f52c..e904a7b60e 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java @@ -123,6 +123,10 @@ protected void configureServlets() { bind(cz.incad.kramerius.rest.apiNew.admin.v10.ItemsResource.class); bind(cz.incad.kramerius.rest.apiNew.admin.v10.ServerFilesResource.class); + + // OAI endpoint + bind(cz.incad.kramerius.rest.oai.OAIEndpoint.class); + bind(KeycloakProxy.class); // debug resource diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientApiResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientApiResource.java index 61823f81aa..10d21ca4b3 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientApiResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientApiResource.java @@ -33,6 +33,18 @@ public abstract class ClientApiResource extends ApiResource { @Named("new-index") SolrAccess solrAccess; + public SolrAccess getSolrAccess() { + return solrAccess; + } + + public RightsResolver getRightsResolver() { + return rightsResolver; + } + + public Provider getUserProvider() { + return userProvider; + } + public void checkCurrentUserByJsessionidIsAllowedToPerformGlobalSecuredAction(SecuredActions action) { User user = this.userProvider.get(); if (user == null || user.getLoginname().equals("not_logged")) { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/DeletedRecordsDetection.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/DeletedRecordsDetection.java new file mode 100644 index 0000000000..bd31c5c72d --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/DeletedRecordsDetection.java @@ -0,0 +1,21 @@ +/* + * Copyright (C) Jan 11, 2024 Pavel Stastny + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cz.incad.kramerius.rest.oai; + +public interface DeletedRecordsDetection { + +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/ErrorCode.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/ErrorCode.java new file mode 100644 index 0000000000..d39d7117db --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/ErrorCode.java @@ -0,0 +1,38 @@ +/* + * Copyright (C) Jan 10, 2024 Pavel Stastny + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cz.incad.kramerius.rest.oai; + +public enum ErrorCode { + + // acceptable for all verbs + badArgument, + // acceptable badResumptionToken + badResumptionToken, + // no verb + badVerb, + // GetRecord, ListIdentifiers, ListRecords + cannotDisseminateFormat, + // GetRecord, ListMetadataFormat + idDoesNotExist, + //ListIdentifiers ListRecords + noRecordsMatch, + //ListMetadataFormats + noMetadataFormats, + + //ListSets ListIdentifiers ListRecords + noSetHierarchy +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java new file mode 100644 index 0000000000..2a01c2a0f7 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java @@ -0,0 +1,270 @@ +/* + * Copyright (C) Jan 10, 2024 Pavel Stastny + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cz.incad.kramerius.rest.oai; + +import java.io.IOException; +import java.net.URL; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.servlet.http.HttpServletRequest; + +import org.w3c.dom.DOMException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import cz.incad.kramerius.FedoraAccess; +import cz.incad.kramerius.FedoraNamespaces; +import cz.incad.kramerius.fedora.om.RepositoryException; +import cz.incad.kramerius.rest.oai.exceptions.OAIException; +import cz.incad.kramerius.utils.ApplicationURL; +import cz.incad.kramerius.utils.XMLUtils; +import cz.incad.kramerius.utils.conf.KConfiguration; + +public enum MetadataExport { + + + oaiDc("oai_dc", + "http://www.openarchives.org/OAI/2.0/oai_dc.xsd", + "http://www.openarchives.org/OAI/2.0/oai_dc/") { + + @Override + public Element perform(HttpServletRequest request, FedoraAccess fa, Document owningDocument, String oaiIdentifier,OAISet set) { + try { + String pid = OAITools.pidFromOAIIdentifier(oaiIdentifier); + Document dc = fa.getDC(pid); + if (dc != null) { + Element rootElement = dc.getDocumentElement(); + owningDocument.adoptNode(rootElement); + return rootElement; + } else return null; + } catch (IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + throw new RuntimeException(e.getMessage()); + } + } + }, + + + edm("edm","http://www.europeana.eu/schemas/ese/","") { + + @Override + public Element perform(HttpServletRequest request, FedoraAccess fa, Document owningDocument, + String oaiIdentifier,OAISet set) { + + try { + + String baseUrl = ApplicationURL.applicationURL(request); + //rdf:about="uuid:6b182ad3-b9e9-11e1-1726-001143e3f55c" + String pid = OAITools.pidFromOAIIdentifier(oaiIdentifier); + Document dc = fa.getDC(pid); + Element dcElement = dc.getDocumentElement(); + + Element metadata = owningDocument.createElement("metadata"); + metadata.setAttribute("xmlns:europeana", "http://www.europeana.eu/schemas/ese/"); + metadata.setAttribute("xmlns:ore", "http://www.openarchives.org/ore/terms/"); + metadata.setAttribute("xmlns:edm", "http://www.europeana.eu/schemas/edm/"); + metadata.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); + metadata.setAttribute("xmlns:rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#"); + metadata.setAttribute("xmlns:rdaGr2", "http://rdvocab.info/ElementsGr2/"); + metadata.setAttribute("xmlns:skos", "http://www.w3.org/2004/02/skos/core#"); + metadata.setAttribute("xmlns:oai_dc", "http://www.openarchives.org/OAI/2.0/oai_dc/"); + metadata.setAttribute("xmlns:dc", "http://purl.org/dc/elements/1.1/"); + metadata.setAttribute("xmlns:dcterms", "http://purl.org/dc/terms/"); + + Element rdf = owningDocument.createElementNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#","rdf:RDF"); + rdf.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",oaiIdentifier); + metadata.appendChild(rdf); + + Element providedCHO = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:ProvidedCHO"); + rdf.appendChild(providedCHO); + + List elements = XMLUtils.getElements(dcElement); + elements.stream().forEach(dcElm -> { + owningDocument.adoptNode(dcElm); + providedCHO.appendChild(dcElm); + }); + + //rdf.appendChild(providedCHO); + + Element type = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:type"); + providedCHO.appendChild(type); + type.setTextContent("TEXT"); + + Element webresource = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:WebResource"); + webresource.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",String.format("%s/api/client/v7.0/items/%s/image", baseUrl, pid)); + metadata.appendChild(webresource); + + Element edmAggregation = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:Aggregation"); + String clientUrl = KConfiguration.getInstance().getConfiguration().getString("client"); + if (clientUrl != null) { + edmAggregation.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); + } else { + edmAggregation.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); + } + Element edmDataPrvovider = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:dataProvider"); + + String edmDataProvider = set.getAdditionalsInfo().get("edm:dataProvider"); + if (edmDataProvider != null) { + edmDataPrvovider.setTextContent(edmDataProvider); + } else { + edmDataPrvovider.setTextContent("Academy of Sciences Library/Knihovna Akademie věd ČR"); + } + edmAggregation.appendChild(edmDataPrvovider); + + // dodat dle setu + Element shownAt = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:isShownAt"); + if (clientUrl != null) { + shownAt.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource",clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); + } else { + shownAt.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource",baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); + } + edmAggregation.appendChild(shownAt); + + // mapovani na licence + Element edmRights = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:rights"); + edmRights.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource", "https://cdk.lib.cas.cz/uuid/"+pid); + edmAggregation.appendChild(edmRights); + + + Element edmObject = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:object"); + edmObject.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource", String.format("%s/api/client/v7.0/items/%s/image", baseUrl, pid)); + edmAggregation.appendChild(edmObject); + + Element edmProvider = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:provider"); + edmProvider.setTextContent("Czech digital library/Česká digitální knihovna"); + edmAggregation.appendChild(edmProvider); + + metadata.appendChild(edmAggregation); + + return metadata; + } catch (IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + throw new RuntimeException(e.getMessage()); + } + } + + }, + + + + /** Disabled in CDK */ +// ese("ese", +// "http://www.europeana.eu/schemas/ese/ESE-V3.2.xsd", +// "http://www.europeana.eu/schemas/ese/") { +// @Override +// public Element perform(HttpServletRequest request, FedoraAccess fa, Document owningDocument, String oaiIdentifier,OAISet set) { +// try { +// String baseUrl = ApplicationURL.applicationURL(request); +// String pid = OAITools.pidFromOAIIdentifier(oaiIdentifier); +// Document dc = fa.getDC(pid); +// Element dcElement = dc.getDocumentElement(); +// +// Element record = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "record"); +// record.setAttribute("xmlns:dc", FedoraNamespaces.DC_NAMESPACE_URI); +// +// record.setAttributeNS("http://www.w3.org/2001/XMLSchema-instance","xsi:schemaLocation","http://www.europeana.eu/schemas/ese/ http://www.europeana.eu/schemas/ese/ESE-V3.2.xsd http://purl.org/dc/elements/1.1/ http://www.dublincore.org/schemas/xmls/qdc/dc.xsd http://purl.org/dc/terms/ http://www.dublincore.org/schemas/xmls/qdc/dcterms.xsd http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd"); +// List dcElems = XMLUtils.getElements(dcElement); +// dcElems.stream().forEach(dcElm-> { +// owningDocument.adoptNode(dcElm); +// record.appendChild(dcElm); +// }); +// +// Element object = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "object"); +// String thumb = String.format(baseUrl+(baseUrl.endsWith("/")? "" : "/")+"api/client/v7.0/items/%s/image/thumb", pid); +// object.setTextContent(thumb); +// record.appendChild(object); +// +// Element provider = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "provider"); +// provider.setTextContent("Academy of Sciences Library"); //TODO: To configuration +// record.appendChild(provider); +// +// Element type = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "type"); +// type.setTextContent("TEXT"); +// record.appendChild(type); +// +// Element isShownAt = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "isShownAt"); +// String clientUrl = KConfiguration.getInstance().getConfiguration().getString("client"); +// if (clientUrl != null) { +// isShownAt.setTextContent(clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); +// } else { +// isShownAt.setTextContent(baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); +// } +// record.appendChild(isShownAt); +// +// +// return record; +// +// } catch (IOException e) { +// LOGGER.log(Level.SEVERE,e.getMessage(),e); +// throw new RuntimeException(e.getMessage()); +// } +// } +// }; + + /** disabled for now */ +// drkramerius4("drkramerius4", +// "http://registrdigitalizace.cz/schemas/drkramerius/v4/drkram.xsd", +// "http://registrdigitalizace.cz/schemas/drkramerius/v4/") { +// @Override +// public Element perform(HttpServletRequest request, FedoraAccess fa, Document owningDocument, String oaiIdentifier) { +// // find rdkramerius +// return null; +// } +// }; + + public static final Logger LOGGER = Logger.getLogger(MetadataExport.class.getName()); + + + private MetadataExport(String metadataPrefix, String schema, String metadataNamespace) { + this.metadataPrefix = metadataPrefix; + this.schema = schema; + this.metadataNamespace = metadataNamespace; + } + + public String getMetadataNamespace() { + return metadataNamespace; + } + + public String getSchema() { + return schema; + } + + public String getMetadataPrefix() { + return metadataPrefix; + } + + + public static MetadataExport findByPrefix(String prefix) { + MetadataExport[] values = MetadataExport.values(); + for (MetadataExport me : values) { + if (me.getMetadataPrefix().equals(prefix)) return me; + } + return null; + } + + private String metadataPrefix; + private String schema; + private String metadataNamespace; + + + public abstract Element perform(HttpServletRequest request, FedoraAccess fa, Document owningDocument, String oaiIdentifier, OAISet set); + +// public static final Logger LOGGER = Logger.getLogger(MetadataExport.class.getName()); + +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIEndpoint.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIEndpoint.java new file mode 100644 index 0000000000..1416a7d61c --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIEndpoint.java @@ -0,0 +1,107 @@ +/* + * Copyright (C) Jan 10, 2024 Pavel Stastny + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cz.incad.kramerius.rest.oai; + +import javax.inject.Inject; + +import javax.inject.Named; +import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.GET; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.Context; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.PathSegment; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.UriInfo; + +import org.json.JSONObject; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import com.google.inject.Provider; + +import cz.incad.kramerius.FedoraAccess; +import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.repository.KrameriusRepositoryApiImpl; +import cz.incad.kramerius.rest.apiNew.client.v60.ClientApiResource; +import cz.incad.kramerius.rest.apiNew.exceptions.InternalErrorException; +import cz.incad.kramerius.rest.oai.exceptions.OAIException; +import cz.incad.kramerius.utils.ApplicationURL; +import cz.incad.kramerius.utils.StringUtils; +import cz.incad.kramerius.utils.XMLUtils; +import cz.incad.kramerius.utils.conf.KConfiguration; + +import cz.incad.kramerius.SolrAccess; + + +import static cz.incad.kramerius.rest.oai.OAITools.*; + + +import java.io.StringWriter; +import java.util.logging.Level; +import java.util.logging.Logger; + +@Path("/harvest/v7.0") +public class OAIEndpoint extends ClientApiResource { + + public static Logger LOGGER = Logger.getLogger(OAIEndpoint.class.getName()); + + + @Inject + @Named("cachedFedoraAccess") + private transient FedoraAccess fedoraAccess; + + @Inject + Provider requestProvider; + + public OAIEndpoint() { + } + + + @GET + @Path("oai") + @Produces(MediaType.APPLICATION_XML) + public Response oai( + @QueryParam("verb") String verb, + @QueryParam("set") String set, + @QueryParam("metadataPrefix") String metadataPrefix + ) throws OAIException { + + if (StringUtils.isAnyString(verb)) { + try { + OAIVerb oaiVerb = OAIVerb.valueOf(verb); + Document oai = createOAIDocument(); + Element oaiRoot = oai.getDocumentElement(); + oaiVerb.perform(this.fedoraAccess, this.getSolrAccess(), this.requestProvider.get(), oai, oaiRoot); + StringWriter writer = new StringWriter(); + XMLUtils.print(oai, writer); + return Response.ok(writer.toString()).build(); + } catch(OAIException e) { + throw e; + } catch (Exception e) { + LOGGER.log(Level.SEVERE, e.getMessage(),e); + throw new OAIException(ErrorCode.badVerb, null,null, ApplicationURL.applicationURL(requestProvider.get()),null); + } + } else { + throw new OAIException(ErrorCode.badVerb, null,null, ApplicationURL.applicationURL(requestProvider.get()),null); + } + } +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java new file mode 100644 index 0000000000..1e78c56895 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java @@ -0,0 +1,118 @@ +/* + * Copyright (C) Jan 14, 2024 Pavel Stastny + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cz.incad.kramerius.rest.oai; + +import java.io.IOException; +import java.io.StringReader; +import java.net.URLEncoder; +import java.time.OffsetDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; +import java.util.stream.Collectors; + +import javax.servlet.http.HttpServletRequest; +import javax.xml.parsers.ParserConfigurationException; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import cz.incad.kramerius.FedoraAccess; +import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.utils.XMLUtils; + +public class OAIRecord { + + private String identifier; + private String solrIdentifier; + + public OAIRecord(String solrIdentifier, String identifier) { + super(); + this.solrIdentifier = solrIdentifier; + this.identifier = identifier; + } + + public String getIdentifier() { + return identifier; + } + + public String getSolrIdentifier() { + return solrIdentifier; + } + + /** find oai record */ + public static OAIRecord findRecord(SolrAccess solrAccess,String oaiIdentifier) throws IOException, ParserConfigurationException, SAXException { + String pid = OAITools.pidFromOAIIdentifier(oaiIdentifier); + + String encodedQuery = URLEncoder.encode(String.format("pid:\"%s\"", pid),"UTF-8"); + String query = String.format("q=%s", encodedQuery); + String solrResponseXml = solrAccess.requestWithSelectReturningString(query, "xml"); + Document document = XMLUtils.parseDocument(new StringReader(solrResponseXml)); + + Element result = XMLUtils.findElement(document.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + return element.getNodeName().equals("result"); + } + }); + + if (result != null) { + List docs = XMLUtils.getElements(result, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + return element.getNodeName().equals("doc"); + } + }); + + if (docs.size() > 0) { + Element pidElm = XMLUtils.findElement(docs.get(0), "str"); + return new OAIRecord(pidElm.getTextContent(), oaiIdentifier); + + } else return null; + } else { + return null; + } + + } + + /** render metadata */ + public Element toMetadata(HttpServletRequest request, FedoraAccess fa, Document doc, MetadataExport export) { + return export.perform(request, fa, doc, identifier); + } + + /** render header */ + public Element toHeader(Document doc, String setSpec) { + Element header = doc.createElement("header"); + + Element identifier = doc.createElement("identifier"); + identifier.setTextContent(this.identifier); + header.appendChild(identifier); + + OffsetDateTime now = OffsetDateTime.now(); + Element datestamp = doc.createElement("datestamp"); + datestamp.setTextContent(now.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + header.appendChild(datestamp); + + if (setSpec != null) { + Element setSpecElm = doc.createElement("setSpec"); + setSpecElm.setTextContent(setSpec); + header.appendChild(setSpecElm); + } + + return header; + } +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIResults.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIResults.java new file mode 100644 index 0000000000..445e51987a --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIResults.java @@ -0,0 +1,53 @@ +/* + * Copyright (C) Jan 15, 2024 Pavel Stastny + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cz.incad.kramerius.rest.oai; + +import java.util.List; + +public class OAIResults { + + private String metadataPrerix; + private int completeListSize; + private String resumptionToken; + private List records; + + public OAIResults(int completeListSize, String resumptionToken, String metadataPrefix, List records) { + super(); + this.completeListSize = completeListSize; + this.resumptionToken = resumptionToken; + this.records = records; + this.metadataPrerix = metadataPrefix; + } + + + public int getCompleteListSize() { + return completeListSize; + } + + public String getResumptionToken() { + return resumptionToken; + } + + + public String getMetadataPrerix() { + return metadataPrerix; + } + + public List getRecords() { + return records; + } +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java new file mode 100644 index 0000000000..cdfd6cb358 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java @@ -0,0 +1,181 @@ +/* + * Copyright (C) Jan 10, 2024 Pavel Stastny + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cz.incad.kramerius.rest.oai; + +import java.io.IOException; +import java.io.StringReader; +import java.io.StringWriter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +import javax.ws.rs.core.UriBuilder; +import javax.xml.parsers.ParserConfigurationException; + +import org.json.JSONObject; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.rest.apiNew.client.v70.SearchResource; +import cz.incad.kramerius.solr.SolrFieldsMapping; +import cz.incad.kramerius.utils.ApplicationURL; +import cz.incad.kramerius.utils.StringUtils; +import cz.incad.kramerius.utils.XMLUtils; + +public class OAISet { + + private String host; + + private String setSpec; + private String setName; + private String setDescription; + private String filterQuery; + + private Map additionalsInfo = new HashMap<>(); + + public OAISet( + String host, + String setSpec, + String setName, + String setDescription, + String filterQuery) { + this(host, setSpec, setName, setDescription, filterQuery, new HashMap<>()); + } + + public OAISet( + String host, + String setSpec, + String setName, + String setDescription, + String filterQuery, Map map) { + this.setSpec = setSpec; + this.setName = setName; + this.setDescription = setDescription; + this.filterQuery = filterQuery; + this.host = host; + this.additionalsInfo = map; + } + + + + + protected OAISet(String host) { + super(); + this.host= host; + } + + public String getHost() { + return host; + } + + public void setHost(String host) { + this.host = host; + } + + public String getSetSpec() { + return setSpec; + } + + public void setSetSpec(String setSpec) { + this.setSpec = setSpec; + } + + public String getSetName() { + return setName; + } + + public void setSetName(String setName) { + this.setName = setName; + } + + public String getSetDescription() { + return setDescription; + } + public void setSetDescription(String setDescription) { + this.setDescription = setDescription; + } + + public String getFilterQuery() { + return filterQuery; + } + + public void setFilterQuery(String filterQuery) { + this.filterQuery = filterQuery; + } + + public Map getAdditionalsInfo() { + return additionalsInfo; + } + + public boolean isMyResumptionToken(String resumptionToken) { + String spec = OAITools.specFromResumptionToken(resumptionToken); + return spec.equals(getSetSpec()); + } + + + public OAIResults findRecords(SolrAccess solrAccess,String cursor, String metadataPrefix, int rows) throws IOException, ParserConfigurationException, SAXException { + + String query = String.format("q=%s&cursorMark=%s&fl=pid&rows=%d&sort=pid+asc", this.filterQuery, cursor, rows); + String solrResponseXml = solrAccess.requestWithSelectReturningString(query, "xml"); + Document document = XMLUtils.parseDocument(new StringReader(solrResponseXml)); + + Element result = XMLUtils.findElement(document.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + return element.getNodeName().equals("result"); + } + }); + + if (result != null) { + + String number = result.getAttribute("numFound"); + + String solrNextCursor = null; + Element cursorElement = XMLUtils.findElement(document.getDocumentElement(), (element) -> { + if (element.getNodeName().equals("str") && element.getAttribute("name").equals("nextCursorMark")) { + return true; + } else return false; + }); + + if (cursorElement != null) { + solrNextCursor = cursorElement.getTextContent(); + } + + List docs = XMLUtils.getElements(result, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + return element.getNodeName().equals("doc"); + } + }); + + List records = docs.stream().map(doc-> { + Element pidElm = XMLUtils.findElement(doc, "str"); + String oaiIdentifier = OAITools.oaiIdentfier(host, pidElm.getTextContent()); + return new OAIRecord(pidElm.getTextContent(), oaiIdentifier); + }).collect(Collectors.toList()); + + String nextCursor = records.size() == rows && solrNextCursor != null ? solrNextCursor+":"+this.setSpec+":"+metadataPrefix : null; + OAIResults results = new OAIResults(Integer.parseInt(number), nextCursor, metadataPrefix, records); + return results; + } else { + return null; + } + } +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISets.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISets.java new file mode 100644 index 0000000000..9b5394ae05 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISets.java @@ -0,0 +1,153 @@ +/* + * Copyright (C) Jan 11, 2024 Pavel Stastny + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cz.incad.kramerius.rest.oai; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.logging.Logger; + +import cz.incad.kramerius.utils.conf.KConfiguration; + +public class OAISets { + + private OAISet defaultSet; + private List sets = new ArrayList<>(); + + public OAISets(String host) { + + /* Default sets are disabled + sets.addAll( + Arrays.asList( + + new OAISet( host, "monograph", + "Set of monographs", + "-", "model:monograph"), + + new OAISet(host,"periodical", + "Set of periodicals", "-", + "model:periodical"), + + new OAISet(host,"periodicalitem", + "Set of periodical items", + "-", "model:periodicalitem"), + + new OAISet(host,"periodicalvolume", + "Set of periodical volumes", + "-", "model:periodicalvolume"), + + new OAISet(host,"manuscript", + "Set of periodical manuscripts", + "-", "model:manuscript"), + + new OAISet(host,"graphic", + "Set of periodical graphics", + "-", "model:graphic"), + + new OAISet(host,"map", + "Set of periodical maps", + "-", "model:map"), + + new OAISet(host,"sheetmusic", + "Set of periodical sheetmusics", + "-", "model:sheetmusic"), + + new OAISet(host,"article", + "Set of periodical articles", + "-", "model:article"), + + new OAISet(host,"supplement", + "Set of periodical supplements", + "-", "model:supplement") + )); + */ + + + this.defaultSet = this.sets.get(0); + // configuration for new sets + Map configuredSets = new HashMap(); + Iterator keys = KConfiguration.getInstance().getConfiguration().getKeys("oai.set"); + while(keys.hasNext()) { + String key = keys.next(); + String rest = key.substring("oai.set.".length()); + String[] values = rest.split("\\."); + if (values.length == 2) { + String spec = values[0]; + if (!configuredSets.containsKey(spec)) { + configuredSets.put(spec, new OAISet(host)); + configuredSets.get(spec).setSetSpec(spec); + } + String property = values[1]; + switch(property) { + case "name":{ + OAISet set = configuredSets.get(spec); + set.setSetName(KConfiguration.getInstance().getProperty(key)); + } + break; + case "desc": + case "description":{ + OAISet set = configuredSets.get(spec); + set.setSetDescription(KConfiguration.getInstance().getProperty(key)); + } + break; + case "filter":{ + OAISet set = configuredSets.get(spec); + set.setFilterQuery(KConfiguration.getInstance().getProperty(key)); + } + break; + default: { + OAISet set = configuredSets.get(spec); + set.getAdditionalsInfo().put(key, KConfiguration.getInstance().getProperty(key)); + } + break; + } + } + } + configuredSets.values().stream().filter(oai -> oai.getFilterQuery() != null).forEach(sets::add); + LOGGER.info("OAI -> Configured sets :"+this.sets); + } + + public OAISet findBySet(String setName) { + Optional found = this.sets.stream() + .filter(oaiSet -> oaiSet.getSetSpec().equals(setName)) + .findFirst(); + return found.isPresent() ? found.get() : null; + } + + public OAISet findByToken(String token) { + Optional found = this.sets.stream() + .filter(oaiSet -> oaiSet.isMyResumptionToken(token)) + .findFirst(); + return found.get(); + } + + + + public List getAOISets() { + return this.sets; + } + + public OAISet getDefaultSet() { + return defaultSet; + } + + public static final Logger LOGGER = Logger.getLogger(OAISets.class.getName()); +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAITools.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAITools.java new file mode 100644 index 0000000000..399128c79e --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAITools.java @@ -0,0 +1,133 @@ +/* + * Copyright (C) Jan 10, 2024 Pavel Stastny + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cz.incad.kramerius.rest.oai; + +import java.time.LocalDateTime; +import java.time.OffsetDateTime; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; + +import javax.xml.XMLConstants; +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; + +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import cz.incad.kramerius.utils.XMLUtils; + +public class OAITools { + + private static final String XSI_SCHEMA_LOCATION = "http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"; + private static final String OAI_ARCHIVES = "http://www.openarchives.org/OAI/2.0/"; + private static final String XSI_NAMESPACE = "http://www.w3.org/2001/XMLSchema-instance"; + + + public static Document createOAIDocument() throws ParserConfigurationException { + DocumentBuilderFactory builderFactory = DocumentBuilderFactory.newInstance(); + DocumentBuilder docBuilder = builderFactory.newDocumentBuilder(); + Document document = docBuilder.newDocument(); + + // Definujte jmenné prostory a jejich přidání k elementům + String xsiNamespace = XSI_NAMESPACE; + String oaiNamespace = OAI_ARCHIVES; + + Element oaiPmhElement = document.createElementNS(oaiNamespace, "OAI-PMH"); + oaiPmhElement.setAttribute("xmlns:xsi", xsiNamespace); + oaiPmhElement.setAttribute("xsi:schemaLocation", XSI_SCHEMA_LOCATION); + + Element responseDateElement = document.createElement("responseDate"); + + OffsetDateTime now = OffsetDateTime.now(); + + responseDateElement.setTextContent(now.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + oaiPmhElement.appendChild(responseDateElement); + + document.appendChild(oaiPmhElement); + return document; + } + + public static Element requestElement(Document doc, OAIVerb verb, OAISet set, String baseUrl, MetadataExport metadata) { + Element elm = doc.createElement( "request"); + if (verb != null) { + elm.setAttribute( "verb", verb.name()); + } + if (set != null) { + elm.setAttribute( "set", set.getSetSpec()); + } + if (metadata != null) { + elm.setAttribute( "metadataPrefix", metadata.getMetadataPrefix()); + } + elm.setTextContent(baseUrl+(baseUrl.endsWith("/")? "" : "/")+"api/harvest/v7.0/oai"); + return elm; + } + + /** Medata from resumption token */ + public static String metadataFromResumptionToken(String resumptionToken) { + String[] split = resumptionToken.split(":"); + if (split.length > 2) { + return split[2]; + } else return null; + } + + /** Set spec from resumption token */ + public static String specFromResumptionToken(String resumptionToken) { + if (resumptionToken.contains(":")) { + String[] split = resumptionToken.split(":"); + if (split.length > 1) { + return split[1]; + } else return null; + + } else return null; + } + + /** Solr cursorMark from resumption token */ + public static String solrCursorMarkFromResumptionToken(String resumptionToken) { + if (resumptionToken.contains(":")) { + String[] split = resumptionToken.split(":"); + if (split.length > 1) { + return split[0]; + } else return null; + } else return null; + } + + /** create OAI identifier */ + public static String oaiIdentfier(String host, String pid) { + String oaiIdentifier = String.format("oai:%s:%s", host, pid); + return oaiIdentifier; + } + + + /** host from oai identifier */ + public static String hostFromOAIIdentifier(String oaiIdentifier) { + String[] split = oaiIdentifier.split(":"); + if (split.length > 1) { + return split[1]; + } else return null; + } + + /** pid from oai identifier */ + public static String pidFromOAIIdentifier(String oaiIdentifier) { + int pidIndex = oaiIdentifier.indexOf("uuid:"); + if (pidIndex > -1) { + return oaiIdentifier.substring(pidIndex); + } else return null; + } + +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java new file mode 100644 index 0000000000..49cf601619 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java @@ -0,0 +1,439 @@ +/* + * Copyright (C) Jan 10, 2024 Pavel Stastny + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cz.incad.kramerius.rest.oai; + +import java.io.IOException; +import java.net.MalformedURLException; +import java.net.URL; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.servlet.http.HttpServletRequest; +import javax.xml.parsers.ParserConfigurationException; + +import org.w3c.dom.DOMException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import cz.incad.kramerius.FedoraAccess; +import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.rest.oai.exceptions.OAIException; +import cz.incad.kramerius.utils.ApplicationURL; +import cz.incad.kramerius.utils.XMLUtils; +import cz.incad.kramerius.utils.conf.KConfiguration; + +public enum OAIVerb { + // metadata formats + ListMetadataFormats { + @Override + public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException{ + + Element requestElement = OAITools.requestElement(doc, OAIVerb.ListMetadataFormats,null,ApplicationURL.applicationURL(request),null); + doc.getDocumentElement().appendChild(requestElement); + + Element listMetadataPrefix = doc.createElement("ListMetadataFormats"); + doc.getDocumentElement().appendChild(listMetadataPrefix); + MetadataExport[] values = MetadataExport.values(); + for (MetadataExport metadataExport : values) { + Element metadataFormat= doc.createElement("metadataFormat"); + + Element metadataPrefix = doc.createElement("metadataPrefix"); + metadataPrefix.setTextContent(metadataExport.getMetadataPrefix()); + metadataFormat.appendChild(metadataPrefix); + + Element schema = doc.createElement("schema"); + schema.setTextContent(metadataExport.getSchema()); + metadataFormat.appendChild(schema); + + Element metadataNamespace = doc.createElement("metadataNamespace"); + metadataNamespace.setTextContent(metadataExport.getMetadataNamespace()); + metadataFormat.appendChild(metadataNamespace); + + listMetadataPrefix.appendChild(metadataFormat); + } + + doc.getDocumentElement().appendChild(listMetadataPrefix); + + } + }, + ListSets { + @Override + public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException{ + try { + Element requestElement = OAITools.requestElement(doc, OAIVerb.ListSets, null,ApplicationURL.applicationURL(request), null); + doc.getDocumentElement().appendChild(requestElement); + + String baseUrl = ApplicationURL.applicationURL(request); + URL urlObject = new URL(baseUrl); + + OAISets sets = new OAISets(urlObject.getHost()); + + Element listSets = doc.createElement("ListSets"); + doc.getDocumentElement().appendChild(listSets); + List oaiSets = sets.getAOISets(); + + for (OAISet oaiIterationSet : oaiSets) { + Element setDefinition= doc.createElement("set"); + + Element setSpec = doc.createElement("setSpec"); + setSpec.setTextContent(oaiIterationSet.getSetSpec()); + setDefinition.appendChild(setSpec); + + Element setName = doc.createElement("setName"); + setName.setTextContent(oaiIterationSet.getSetName()); + setDefinition.appendChild(setName); + + Element setDescription = doc.createElement("setDescription"); + if (oaiIterationSet.getSetDescription() != null) { + setDescription.setTextContent(oaiIterationSet.getSetDescription()); + } + setDefinition.appendChild(setDescription); + + listSets.appendChild(setDefinition); + } + + doc.getDocumentElement().appendChild(listSets); + } catch (MalformedURLException | DOMException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + throw new OAIException(ErrorCode.badArgument, OAIVerb.ListIdentifiers,null, ApplicationURL.applicationURL(request),null); + } + } + }, + + Identify { + + @Override + public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException { + + try { + String url = ApplicationURL.applicationURL(request); + + Element requestElement = OAITools.requestElement(doc, OAIVerb.Identify, null,ApplicationURL.applicationURL(request), null); + doc.getDocumentElement().appendChild(requestElement); + + + String oaiName = KConfiguration.getInstance().getConfiguration().getString(REPOSITORY_NAME,"kramerius"); + String baseUrl = KConfiguration.getInstance().getConfiguration().getString(REPOSITORY_BASE_URL,url); + String adminEmail = KConfiguration.getInstance().getConfiguration().getString(REPOSITORY_ADMIN_EMAIL,"-none-"); + String earliestDateTimestamp = KConfiguration.getInstance().getConfiguration().getString("oai.earliestDatestamp","2012-06-30T22:26:40Z"); + + URL urlObject = new URL(baseUrl); + + + String protocolVersion = KConfiguration.getInstance().getConfiguration().getString("oai.protocolVersion","2.0"); + String deletedRecord = KConfiguration.getInstance().getConfiguration().getString("oai.deletedRecord","transient"); + String granularity = KConfiguration.getInstance().getConfiguration().getString("oai.granularity","YYYY-MM-DDThh:mm:ssZ"); + + Element identify = doc.createElement("Identify"); + + Element repositoryNameElm = doc.createElement("repositoryName"); + repositoryNameElm.setTextContent(oaiName); + identify.appendChild(repositoryNameElm); + + Element baseURLElm = doc.createElement("baseURL"); + baseURLElm.setTextContent(baseUrl); + identify.appendChild(baseURLElm); + + Element protocolVersionElm = doc.createElement("protocolVersion"); + protocolVersionElm.setTextContent(protocolVersion); + identify.appendChild(protocolVersionElm); + + Element adminEmailElm = doc.createElement("adminEmail"); + adminEmailElm.setTextContent(adminEmail); + identify.appendChild(adminEmailElm); + + Element earliestDatestampElm = doc.createElement("earliestDatestamp"); + earliestDatestampElm.setTextContent(earliestDateTimestamp); + identify.appendChild(earliestDatestampElm); + + Element deleteRecordsElm = doc.createElement("deletedRecord"); + deleteRecordsElm.setTextContent(deletedRecord); + identify.appendChild(deleteRecordsElm); + + Element granularityElm = doc.createElement("granularity"); + granularityElm.setTextContent(granularity); + identify.appendChild(granularityElm); + + Element descriptionElm = doc.createElement("description"); + identify.appendChild(descriptionElm); + + Element oaiIdentifierElm = doc.createElementNS("http://www.openarchives.org/OAI/2.0/oai-identifier", "oai-identifier"); + descriptionElm.appendChild(oaiIdentifierElm); + + Element schemeElm = doc.createElement("scheme"); + schemeElm.setTextContent("oai"); + descriptionElm.appendChild(schemeElm); + + Element repositoryIdentifierElm = doc.createElement("repositoryIdentifier"); + repositoryIdentifierElm.setTextContent(urlObject.getHost()); + descriptionElm.appendChild(schemeElm); + + Element delimiterElm = doc.createElement("delimiter"); + delimiterElm.setTextContent(":"); + descriptionElm.appendChild(delimiterElm); + + Element sampleIdentifierElm = doc.createElement("sampleIdentifier"); + sampleIdentifierElm.setTextContent("oai:"+urlObject.getHost()+":uuid:530719f5-ee95-4449-8ce7-12b0f4cadb22"); + descriptionElm.appendChild(sampleIdentifierElm); + + doc.getDocumentElement().appendChild(identify); + } catch (MalformedURLException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + throw new OAIException(ErrorCode.badArgument, OAIVerb.ListIdentifiers,null, ApplicationURL.applicationURL(request),null); + } + } + + }, + + ListRecords { + @Override + public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException{ + + OAISet selectedSet = null; + MetadataExport selectedMetadata = null; + try { + String baseUrl = ApplicationURL.applicationURL(request); + + + URL urlObject = new URL(baseUrl); + OAISets sets = new OAISets(urlObject.getHost()); + String set = request.getParameter("set"); + String resumptionToken = request.getParameter("resumptionToken"); + String metadataPrefix = request.getParameter("metadataPrefix"); + if (metadataPrefix != null || resumptionToken != null) { + int rows = KConfiguration.getInstance().getConfiguration().getInt(REPOSITORY_ROWS_IN_RESULTS,600); + if (set != null) { + selectedSet = sets.findBySet(set); + if (selectedSet ==null) { + throw new OAIException(ErrorCode.badArgument, OAIVerb.ListRecords, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); + + } + } else if (resumptionToken != null){ + selectedSet = sets.findByToken(resumptionToken); + metadataPrefix = OAITools.metadataFromResumptionToken(resumptionToken); + if (selectedSet == null || metadataPrefix == null || MetadataExport.findByPrefix(metadataPrefix) == null) { + throw new OAIException(ErrorCode.badResumptionToken, OAIVerb.ListRecords, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); + } + } + + selectedMetadata = MetadataExport.findByPrefix(metadataPrefix); + + if (selectedMetadata != null) { + if (selectedSet == null) { + selectedSet = sets.getDefaultSet(); + } + + Element requestElement = OAITools.requestElement(doc, OAIVerb.ListIdentifiers, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); + doc.getDocumentElement().appendChild(requestElement); + + Element identify = doc.createElement("ListRecords"); + OAIResults results = null; + + if (resumptionToken != null) { + String solrCursor = OAITools.solrCursorMarkFromResumptionToken(resumptionToken); + results = selectedSet.findRecords(solrAccess, solrCursor,metadataPrefix,rows); + for (OAIRecord oaiRec : results.getRecords()) { + + Element record= doc.createElement("record"); + Element header = oaiRec.toHeader(doc, selectedSet.getSetSpec()); + + + Element metadata = doc.createElement("metadata"); + metadata.appendChild(oaiRec.toMetadata(request, fa, doc, selectedMetadata)); + + record.appendChild(header); + record.appendChild(metadata); + + identify.appendChild(record); + + + } + } else { + results = selectedSet.findRecords(solrAccess,"*", metadataPrefix,rows); + for (OAIRecord oaiRec : results.getRecords()) { + + Element record= doc.createElement("record"); + Element header = oaiRec.toHeader(doc, selectedSet.getSetSpec()); + + + Element metadata = doc.createElement("metadata"); + metadata.appendChild(oaiRec.toMetadata(request, fa, doc, selectedMetadata)); + + record.appendChild(header); + record.appendChild(metadata); + + identify.appendChild(record); + } + } + + if (results.getResumptionToken()!= null) { + Element resToken = doc.createElement("resumptionToken"); + resToken.setAttribute("completeListSize", ""+results.getCompleteListSize()); + resToken.setTextContent(results.getResumptionToken()); + identify.appendChild(resToken); + } + + doc.getDocumentElement().appendChild(identify); + } else { + throw new OAIException(ErrorCode.noMetadataFormats, OAIVerb.ListIdentifiers, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); + } + + } else { + throw new OAIException(ErrorCode.noMetadataFormats, OAIVerb.ListIdentifiers, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); + } + + } catch ( IOException | SAXException | ParserConfigurationException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + throw new OAIException(ErrorCode.badArgument, OAIVerb.ListIdentifiers,selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); + } + } + }, + ListIdentifiers { + @Override + public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException{ + + OAISet selectedSet = null; + MetadataExport selectedMetadata = null; + try { + String baseUrl = ApplicationURL.applicationURL(request); + + + URL urlObject = new URL(baseUrl); + OAISets sets = new OAISets(urlObject.getHost()); + String set = request.getParameter("set"); + String resumptionToken = request.getParameter("resumptionToken"); + String metadataPrefix = request.getParameter("metadataPrefix"); + if (metadataPrefix != null || resumptionToken != null) { + int rows = KConfiguration.getInstance().getConfiguration().getInt(REPOSITORY_ROWS_IN_RESULTS,600); + + if (set != null) { + selectedSet = sets.findBySet(set); + } else if (resumptionToken != null){ + selectedSet = sets.findByToken(resumptionToken); + metadataPrefix = OAITools.metadataFromResumptionToken(resumptionToken); + if (selectedSet == null || metadataPrefix == null || MetadataExport.findByPrefix(metadataPrefix) == null) { + throw new OAIException(ErrorCode.badResumptionToken, OAIVerb.ListIdentifiers, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); + } + } + + selectedMetadata = MetadataExport.findByPrefix(metadataPrefix); + + if (selectedMetadata != null) { + if (selectedSet == null) { + selectedSet = sets.getDefaultSet(); + } + + Element requestElement = OAITools.requestElement(doc, OAIVerb.ListIdentifiers, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); + doc.getDocumentElement().appendChild(requestElement); + + Element identify = doc.createElement("ListIdentifiers"); + OAIResults results = null; + if (resumptionToken != null) { + String solrCursor = OAITools.solrCursorMarkFromResumptionToken(resumptionToken); + results = selectedSet.findRecords(solrAccess, solrCursor,metadataPrefix,rows); + for (OAIRecord oaiRec : results.getRecords()) { identify.appendChild(oaiRec.toHeader(doc, selectedSet.getSetSpec()));} + } else { + results = selectedSet.findRecords(solrAccess,"*", metadataPrefix,rows); + for (OAIRecord oaiRec : results.getRecords()) { identify.appendChild(oaiRec.toHeader(doc, selectedSet.getSetSpec()));} + } + + if (results.getResumptionToken()!= null) { + Element resToken = doc.createElement("resumptionToken"); + resToken.setAttribute("completeListSize", ""+results.getCompleteListSize()); + resToken.setTextContent(results.getResumptionToken()); + identify.appendChild(resToken); + } + + doc.getDocumentElement().appendChild(identify); + } else { + throw new OAIException(ErrorCode.noMetadataFormats, OAIVerb.ListIdentifiers, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); + } + + } else { + throw new OAIException(ErrorCode.noMetadataFormats, OAIVerb.ListIdentifiers, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); + } + + } catch ( IOException | SAXException | ParserConfigurationException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + throw new OAIException(ErrorCode.badArgument, OAIVerb.ListIdentifiers,selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); + } + } + }, + GetRecord { + @Override + public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException{ + + MetadataExport selectedMetadata = null; + try { + String baseUrl = ApplicationURL.applicationURL(request); + + String identifier = request.getParameter("Identifier"); + String metadataPrefix = request.getParameter("metadataPrefix"); + if (metadataPrefix == null || MetadataExport.findByPrefix(metadataPrefix) == null) { + throw new OAIException(ErrorCode.cannotDisseminateFormat, OAIVerb.GetRecord, null, ApplicationURL.applicationURL(request),selectedMetadata); + } + + selectedMetadata = MetadataExport.findByPrefix(metadataPrefix); + + if (selectedMetadata != null) { + + Element requestElement = OAITools.requestElement(doc, OAIVerb.GetRecord, null, ApplicationURL.applicationURL(request),selectedMetadata); + doc.getDocumentElement().appendChild(requestElement); + + Element identify = doc.createElement("GetRecord"); + OAIRecord oaiRec = OAIRecord.findRecord(solrAccess,identifier); + if (oaiRec != null) { + + Element record= doc.createElement("record"); + Element header = oaiRec.toHeader(doc, null); + + + Element metadata = doc.createElement("metadata"); + metadata.appendChild(oaiRec.toMetadata(request, fa, doc, selectedMetadata)); + + record.appendChild(header); + record.appendChild(metadata); + + identify.appendChild(record); + + } else { + throw new OAIException(ErrorCode.idDoesNotExist, OAIVerb.GetRecord, null, ApplicationURL.applicationURL(request),selectedMetadata); + } + doc.getDocumentElement().appendChild(identify); + } else { + throw new OAIException(ErrorCode.cannotDisseminateFormat, OAIVerb.GetRecord, null, ApplicationURL.applicationURL(request),selectedMetadata); + } + + } catch ( IOException | SAXException | ParserConfigurationException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + throw new OAIException(ErrorCode.badArgument, OAIVerb.GetRecord,null, ApplicationURL.applicationURL(request),selectedMetadata); + } + + } + }; + + private static final String REPOSITORY_ADMIN_EMAIL = "oai.adminEmail"; + private static final String REPOSITORY_BASE_URL = "oai.baseUrl"; + private static final String REPOSITORY_NAME = "oai.repositoryName"; + private static final String REPOSITORY_ROWS_IN_RESULTS = "oai.rowsInResults"; + + public abstract void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws Exception; + + public static Logger LOGGER = Logger.getLogger(OAIVerb.class.getName()); +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/exceptions/OAIException.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/exceptions/OAIException.java new file mode 100644 index 0000000000..110cd8a8e0 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/exceptions/OAIException.java @@ -0,0 +1,122 @@ +/* + * Copyright (C) Jan 12, 2024 Pavel Stastny + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cz.incad.kramerius.rest.oai.exceptions; + +import javax.ws.rs.WebApplicationException; + + +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; + +import org.codehaus.jettison.json.JSONException; +import org.codehaus.jettison.json.JSONObject; +import org.w3c.dom.DOMException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import cz.incad.kramerius.rest.oai.ErrorCode; +import cz.incad.kramerius.rest.oai.MetadataExport; +import cz.incad.kramerius.rest.oai.OAISet; +import cz.incad.kramerius.rest.oai.OAIVerb; +import cz.incad.kramerius.utils.XMLUtils; + +import static cz.incad.kramerius.rest.oai.OAITools.*; + +import java.io.StringWriter; +import java.util.logging.Level; +import java.util.logging.Logger; + +public class OAIException extends WebApplicationException { + + public static final Logger LOGGER = Logger.getLogger(OAIException.class.getName()); + + protected OAIException(Response response) { + super(response); + } + +// public AbstractOAIException(int errorCode) { +// super(errorCode); +// } +// +// public AbstractOAIException(Response.Status status) { +// super(status); +// } + + public OAIException(ErrorCode oaiErrorCode,OAIVerb verb, OAISet set,String baseUrl,MetadataExport metadata) { + this(Response.status(Response.Status.BAD_REQUEST) + .type(MediaType.APPLICATION_XML) + .entity(buildXml(oaiErrorCode, verb, set, baseUrl, metadata)) + .build()); + } + + public OAIException(int errorCode, ErrorCode oaiErrorCode,OAIVerb verb, OAISet set,String baseUrl,MetadataExport metadata) { + this(Response.status(errorCode) + .type(MediaType.APPLICATION_XML) + .entity(buildXml(oaiErrorCode, verb, set, baseUrl, metadata)) + .build()); + } + + private static String buildXml(ErrorCode oaiErrorCode, OAIVerb verb, OAISet set,String baseUrl,MetadataExport metadata) { + try { + Document oai = createOAIDocument(); + Element oaiRoot = oai.getDocumentElement(); + oaiRoot.appendChild(requestElement(oai, verb, set, baseUrl, metadata)); + + Element error = oai.createElement("error"); + error.setAttribute("code", oaiErrorCode.name()); + oaiRoot.appendChild(error); + + StringWriter writer = new StringWriter(); + XMLUtils.print(oai, writer); + return writer.toString(); + + } catch (DOMException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } catch (ParserConfigurationException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } catch (TransformerException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } + return ""; + } + +// public AbstractOAIException(int errorCode, String messageTemplate, Object... messageArgs) { +// this(errorCode, String.format(messageTemplate, messageArgs)); +// } +// +// public AbstractOAIException(Response.Status status, String message) { +// this(status.getStatusCode(), message); +// } +// +// public AbstractOAIException(Response.Status status, String messageTemplate, Object... messageArgs) { +// this(status, String.format(messageTemplate, messageArgs)); +// } +// +// private static String buildErrorJson(String errorMessage) { +// JSONObject json = new JSONObject(); +// try { +// //json.put("error", errorMessage); +// json.put("message", errorMessage); +// } catch (JSONException e) { +// //noting +// } +// return json.toString(); +// } + +} From ab11ec57365577932ae0f6c8a1c204e3afbc779f Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 24 Jan 2024 22:25:24 +0100 Subject: [PATCH 099/235] Fixed metadata clz --- .../java/cz/incad/kramerius/rest/oai/MetadataExport.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java index 2a01c2a0f7..f1e60dc4ef 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java @@ -159,7 +159,7 @@ public Element perform(HttpServletRequest request, FedoraAccess fa, Document own } } - }, + }; @@ -228,7 +228,6 @@ public Element perform(HttpServletRequest request, FedoraAccess fa, Document own // } // }; - public static final Logger LOGGER = Logger.getLogger(MetadataExport.class.getName()); private MetadataExport(String metadataPrefix, String schema, String metadataNamespace) { @@ -265,6 +264,8 @@ public static MetadataExport findByPrefix(String prefix) { public abstract Element perform(HttpServletRequest request, FedoraAccess fa, Document owningDocument, String oaiIdentifier, OAISet set); + public static final Logger LOGGER = Logger.getLogger(MetadataExport.class.getName()); + // public static final Logger LOGGER = Logger.getLogger(MetadataExport.class.getName()); } From 9d480a3e7aeabc869156ccbbe435d40f2b6897d6 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 24 Jan 2024 22:34:26 +0100 Subject: [PATCH 100/235] Changed metadata export --- .../incad/kramerius/rest/oai/MetadataExport.java | 2 +- .../cz/incad/kramerius/rest/oai/OAIRecord.java | 15 +++++++++++++-- .../java/cz/incad/kramerius/rest/oai/OAISet.java | 1 - .../java/cz/incad/kramerius/rest/oai/OAIVerb.java | 6 +++--- 4 files changed, 17 insertions(+), 7 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java index f1e60dc4ef..c9886a32d5 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java @@ -119,7 +119,7 @@ public Element perform(HttpServletRequest request, FedoraAccess fa, Document own } Element edmDataPrvovider = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:dataProvider"); - String edmDataProvider = set.getAdditionalsInfo().get("edm:dataProvider"); + String edmDataProvider = set != null && set.getAdditionalsInfo() != null ? set.getAdditionalsInfo().get("edm:dataProvider") : null; if (edmDataProvider != null) { edmDataPrvovider.setTextContent(edmDataProvider); } else { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java index 1e78c56895..14c46b65a8 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java @@ -21,6 +21,7 @@ import java.net.URLEncoder; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; +import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; @@ -40,6 +41,8 @@ public class OAIRecord { private String identifier; private String solrIdentifier; +// private List possibleSets = new ArrayList<>(); + public OAIRecord(String solrIdentifier, String identifier) { super(); this.solrIdentifier = solrIdentifier; @@ -53,6 +56,14 @@ public String getIdentifier() { public String getSolrIdentifier() { return solrIdentifier; } + +// public List getPossibleSets() { +// return possibleSets; +// } +// +// public void setPossibleSets(List possibleSets) { +// this.possibleSets = possibleSets; +// } /** find oai record */ public static OAIRecord findRecord(SolrAccess solrAccess,String oaiIdentifier) throws IOException, ParserConfigurationException, SAXException { @@ -90,8 +101,8 @@ public boolean acceptElement(Element element) { } /** render metadata */ - public Element toMetadata(HttpServletRequest request, FedoraAccess fa, Document doc, MetadataExport export) { - return export.perform(request, fa, doc, identifier); + public Element toMetadata(HttpServletRequest request, FedoraAccess fa, Document doc, MetadataExport export, OAISet set) { + return export.perform(request, fa, doc, identifier, set); } /** render header */ diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java index cdfd6cb358..3912650cd9 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java @@ -33,7 +33,6 @@ import org.xml.sax.SAXException; import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.rest.apiNew.client.v70.SearchResource; import cz.incad.kramerius.solr.SolrFieldsMapping; import cz.incad.kramerius.utils.ApplicationURL; import cz.incad.kramerius.utils.StringUtils; diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java index 49cf601619..5716a15214 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java @@ -255,7 +255,7 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r Element metadata = doc.createElement("metadata"); - metadata.appendChild(oaiRec.toMetadata(request, fa, doc, selectedMetadata)); + metadata.appendChild(oaiRec.toMetadata(request, fa, doc, selectedMetadata, selectedSet)); record.appendChild(header); record.appendChild(metadata); @@ -273,7 +273,7 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r Element metadata = doc.createElement("metadata"); - metadata.appendChild(oaiRec.toMetadata(request, fa, doc, selectedMetadata)); + metadata.appendChild(oaiRec.toMetadata(request, fa, doc, selectedMetadata, selectedSet)); record.appendChild(header); record.appendChild(metadata); @@ -405,7 +405,7 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r Element metadata = doc.createElement("metadata"); - metadata.appendChild(oaiRec.toMetadata(request, fa, doc, selectedMetadata)); + metadata.appendChild(oaiRec.toMetadata(request, fa, doc, selectedMetadata, null)); record.appendChild(header); record.appendChild(metadata); From e21da0a9cce814dc19b95fc6aad56e35054f664b Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sat, 10 Feb 2024 15:27:12 +0100 Subject: [PATCH 101/235] Issue https://github.com/ceskaexpedice/ceska-digitalni-knihovna/issues/48 --- .../transform/K7SourceToDestTransform.java | 2 +- .../kramerius/rest/apiNew/ConfigManager.java | 155 +++++++ .../apiNew/client/v60/ClientApiResource.java | 8 +- .../apiNew/client/v60/libs/OneInstance.java | 14 + .../DefaultOnePropertiesInstance.java | 27 +- .../DefaultPropertiesInstances.java | 50 ++- .../v60/redirection/ProxyHandlerSupport.java | 2 - .../redirection/item/ProxyItemHandler.java | 6 + .../redirection/item/V5RedirectHandler.java | 16 + .../redirection/item/V7RedirectHandler.java | 18 + .../kramerius/rest/oai/MetadataExport.java | 420 +++++++++++------- .../incad/kramerius/rest/oai/OAIEndpoint.java | 86 +++- .../incad/kramerius/rest/oai/OAIRecord.java | 62 ++- .../incad/kramerius/rest/oai/OAIResults.java | 1 - .../cz/incad/kramerius/rest/oai/OAISet.java | 82 +++- .../cz/incad/kramerius/rest/oai/OAISets.java | 142 ++++-- .../cz/incad/kramerius/rest/oai/OAIVerb.java | 84 ++-- .../rest/oai/db/OAIDBInitializer.java | 67 +++ .../rest/oai/exceptions/OAIInfoException.java | 61 +++ .../incad/kramerius/rest/utils/IIIFUtils.java | 105 +++++ .../cz/incad/Kramerius/StartupServlet.java | 4 + .../database/impl/res/current.db.version | 2 +- 22 files changed, 1151 insertions(+), 263 deletions(-) create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/oai/db/OAIDBInitializer.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/oai/exceptions/OAIInfoException.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/utils/IIIFUtils.java diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java index 41e659703b..53850fa0b1 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java @@ -66,7 +66,7 @@ public K7SourceToDestTransform() { plainValueFields.put("language", Arrays.asList("languages.facet")); plainValueFields.put("datum_str", Arrays.asList("date.str")); plainValueFields.put("dnnt-labels", Arrays.asList("licenses")); - plainValueFields.put("contains-dnnt-labels", Arrays.asList("contains-licenses")); + plainValueFields.put("contains-dnnt-labels", Arrays.asList("contains_licenses")); plainValueFields.put("text_ocr", Arrays.asList("text_ocr")); // all values but first diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/ConfigManager.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/ConfigManager.java index d65ae99d49..590fccb31c 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/ConfigManager.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/ConfigManager.java @@ -1,16 +1,30 @@ package cz.incad.kramerius.rest.apiNew; +import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.DatabaseUtils; +import cz.incad.kramerius.utils.database.JDBCCommand; +import cz.incad.kramerius.utils.database.JDBCQueryTemplate; +import cz.incad.kramerius.utils.database.JDBCTransactionTemplate; +import cz.incad.kramerius.utils.database.JDBCUpdateTemplate; import javax.inject.Inject; import javax.inject.Named; import javax.inject.Provider; + +import org.apache.commons.lang3.tuple.Pair; + import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; /** * Access to database table CONFIG(KEY:STRING,VALUE:STRING) @@ -24,6 +38,21 @@ public class ConfigManager { @Named("kramerius4") private Provider connectionProvider; + + public List getKeysByRegularExpression(String regexp) { + List keys = new JDBCQueryTemplate(this.connectionProvider.get(), true) { + @Override + public boolean handleRow(ResultSet rs, List returnsList) + throws SQLException { + String key = rs.getString("key"); + returnsList.add(key); + return true; + } + }.executeQuery("SELECT key FROM config WHERE key ~ ? ",regexp); + return keys; + } + + public String getProperty(String key) { Connection conn = connectionProvider.get(); if (conn == null) { @@ -47,6 +76,132 @@ public String getProperty(String key) { } } + public void deleteProperty(String key) { + try { + JDBCTransactionTemplate transactions = new JDBCTransactionTemplate(connectionProvider.get(), true); + List commands = new ArrayList<>(); + commands.add(new JDBCCommand() { + + @Override + public Object executeJDBCCommand(Connection con) throws SQLException { + PreparedStatement prepareStatement = con.prepareStatement("delete from CONFIG where key = ?"); + prepareStatement.setString(1, key); + return prepareStatement.executeUpdate(); + } + }); + transactions.updateWithTransaction(commands); + } catch (SQLException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new RuntimeException(e); + } + } + + + public Map getProperties(Set keys) { + String in = "("+keys.stream().map(k-> {return "'"+k+"'";}).collect(Collectors.joining(","))+")"; + + List> pairs = new JDBCQueryTemplate>(this.connectionProvider.get(), true) { + @Override + public boolean handleRow(ResultSet rs, List> returnsList) + throws SQLException { + String key = rs.getString("key"); + String value = rs.getString("value"); + returnsList.add(Pair.of(key, value)); + return true; + } + }.executeQuery(String.format("SELECT * FROM config WHERE key IN %s ",in)); + + Map map = new HashMap<>(); + pairs.stream().forEach(p-> { + map.put(p.getKey(), p.getValue()); + }); + return map; + } + + + public void deleteProperties(Set keys) { + try { + JDBCTransactionTemplate transactions = new JDBCTransactionTemplate(connectionProvider.get(), true); + List commands = new ArrayList<>(); + + + keys.forEach(key-> { + commands.add(new JDBCCommand() { + @Override + public Object executeJDBCCommand(Connection con) throws SQLException { + // update + String SQL = "DELETE FROM CONFIG WHERE key=?;"; + PreparedStatement pstmt = con.prepareStatement(SQL); + pstmt.setString(1, key); + pstmt.executeUpdate(); + return null; + } + }); + }); + transactions.updateWithTransaction(commands); + } catch (SQLException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new RuntimeException(e); + } + } + + + + + + public void setProperties(Map props) { + try { + JDBCTransactionTemplate transactions = new JDBCTransactionTemplate(connectionProvider.get(), true); + List commands = new ArrayList<>(); + commands.add(new JDBCCommand() { + + + @Override + public Object executeJDBCCommand(Connection con) throws SQLException { + List keys = new JDBCQueryTemplate(con,false) { + @Override + public boolean handleRow(ResultSet rs, List returnsList) throws SQLException { + return returnsList.add(rs.getString("key")); + } + }.executeQuery("select key from config"); + return keys; + } + }); + + props.keySet().forEach(key-> { + commands.add(new JDBCCommand() { + + @Override + public Object executeJDBCCommand(Connection con) throws SQLException { + List keys = (List) getPreviousResult(); + if (keys.contains(key)) { + // update + String SQL = "UPDATE config SET value=? WHERE key=?;"; + PreparedStatement pstmt = con.prepareStatement(SQL); + pstmt.setString(1, props.get(key)); + pstmt.setString(2, key); + pstmt.executeUpdate(); + } else { + // insert + String SQL = "INSERT INTO config(key, value) VALUES (?,?)"; + PreparedStatement pstmt = con.prepareStatement(SQL); + pstmt.setString(1, key); + pstmt.setString(2, props.get(key)); + pstmt.executeUpdate(); + } + return keys; + } + }); + }); + + transactions.updateWithTransaction(commands); + } catch (SQLException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new RuntimeException(e); + } + + } + public void setProperty(String key, String value) { Connection conn = connectionProvider.get(); if (conn == null) { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientApiResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientApiResource.java index 10d21ca4b3..22b9062398 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientApiResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ClientApiResource.java @@ -19,19 +19,19 @@ public abstract class ClientApiResource extends ApiResource { @Inject - Provider userProvider; + protected Provider userProvider; // basic rights resolver / images @Inject - RightsResolver rightsResolver; + protected RightsResolver rightsResolver; @Inject @Named("cachedRightsResolver") - RightsResolver cachedRightsResolver; + protected RightsResolver cachedRightsResolver; @Inject @Named("new-index") - SolrAccess solrAccess; + protected SolrAccess solrAccess; public SolrAccess getSolrAccess() { return solrAccess; diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/OneInstance.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/OneInstance.java index ccc519c930..f25ab533d2 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/OneInstance.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/OneInstance.java @@ -1,5 +1,7 @@ package cz.incad.kramerius.rest.apiNew.client.v60.libs; +import java.util.Map; + import com.sun.jersey.api.client.Client; import cz.incad.kramerius.SolrAccess; @@ -13,6 +15,11 @@ */ public interface OneInstance { + public static final String NAME_CZE = "name_cze"; + public static final String NAME_ENG = "name_eng"; + + + /** type of switch */ public static enum TypeOfChangedStatus { user, automat; @@ -42,6 +49,13 @@ public static InstanceType load(String lt) { public ProxyItemHandler createProxyItemHandler(User user, Client client, SolrAccess solrAccess, String source, String pid,String remoteAddr); public ProxyUserHandler createProxyUserHandler(User user, Client client, SolrAccess solrAccess, String source, String remoteAddr); + + + public Map getRegistrInfo(); + + public void setRegistrInfo(String key, String value); + + public void removeRegistrInfo(String key); //public ProxyHandler createNoPidProxyHandler(User user, Client client, SolrAccess solrAccess, String source, String remoteAddr); } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultOnePropertiesInstance.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultOnePropertiesInstance.java index 4ab6a9de29..9a870427df 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultOnePropertiesInstance.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultOnePropertiesInstance.java @@ -1,5 +1,8 @@ package cz.incad.kramerius.rest.apiNew.client.v60.libs.properties; +import java.util.HashMap; +import java.util.Map; + import com.sun.jersey.api.client.Client; import cz.incad.kramerius.SolrAccess; @@ -18,12 +21,14 @@ import cz.incad.kramerius.utils.conf.KConfiguration; public class DefaultOnePropertiesInstance implements OneInstance { - + private Instances instances; private String instanceAcronym; private boolean connected = true; private TypeOfChangedStatus typeOfChangedStatus = TypeOfChangedStatus.automat; - + + private Map info = new HashMap<>(); + public DefaultOnePropertiesInstance(Instances instances, String instanceAcronym) { super(); this.instanceAcronym = instanceAcronym; @@ -90,6 +95,24 @@ public ProxyUserHandler createProxyUserHandler(User user, Client client, SolrAcc throw new UnsupportedOperationException("supported only for full access"); } } + + + + + @Override + public Map getRegistrInfo() { + return this.info; + } + + @Override + public void setRegistrInfo(String key, String value) { + this.info.put(key, value); + } + + @Override + public void removeRegistrInfo(String key) { + this.info.remove(key); + } @Override public boolean isConnected() { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java index e1f04dfbe8..db49471c13 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java @@ -33,6 +33,10 @@ public class DefaultPropertiesInstances implements Instances { + public static final String INFO_URL = "https://api.registr.digitalniknihovna.cz/api/libraries"; + private static final String STATUS_URL = "https://api.registr.digitalniknihovna.cz/api/libraries?detail=status"; + + public static final Logger LOGGER = Logger.getLogger(DefaultPropertiesInstances.class.getName()); private List instances = new ArrayList<>(); @@ -129,9 +133,11 @@ public boolean isEnabledInstance(String acronym) { public void cronRefresh() { this.refreshingConfiguration(); try { + + Map statuses = new HashMap<>(); Client client = Client.create(); - String string = registerData(client); + String string = registerData(client, STATUS_URL); JSONArray jsonArray = new JSONArray(string); for (int i = 0, ll = jsonArray.length(); i < ll; i++) { JSONObject oneObject = jsonArray.getJSONObject(i); @@ -139,8 +145,28 @@ public void cronRefresh() { Boolean status = oneObject.optBoolean("alive"); statuses.put(code, status); } + + Map> info = new HashMap<>(); + String infoString = registerData(client, INFO_URL); + JSONArray infoArray = new JSONArray(infoString); + for (int i = 0, ll = infoArray.length(); i < ll; i++) { + JSONObject oneObject = infoArray.getJSONObject(i); + String czeName = oneObject.optString("name"); + String engName = oneObject.optString("name_en"); + String code = oneObject.optString("code"); + if (code != null) { + if (!info.containsKey(code)) { + info.put(code, new HashMap<>()); + } + info.get(code).put(OneInstance.NAME_CZE, czeName); + info.get(code).put(OneInstance.NAME_ENG, engName); + + } + } + for (OneInstance oneInstance : instances) { + // statuses boolean isConnected = oneInstance.isConnected(); TypeOfChangedStatus type = oneInstance.getType(); if (type.equals(TypeOfChangedStatus.user) && !isConnected) { @@ -153,16 +179,34 @@ public void cronRefresh() { } } } + + if (info.containsKey(oneInstance.getName())) { + Map instInfo = info.get(oneInstance.getName()); + instInfo.keySet().forEach(key-> { + oneInstance.setRegistrInfo(key, instInfo.get(key)); + }); + } + } } catch (UniformInterfaceException | ClientHandlerException | JSONException | IOException e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); } } - protected String registerData(Client client) throws IOException { - WebResource r = client.resource("https://api.registr.digitalniknihovna.cz/api/libraries?detail=status"); + + + + + + protected String registerData(Client client, String url) throws IOException { + WebResource r = client.resource(url); InputStream inputStream = r.accept(MediaType.APPLICATION_XML).get(InputStream.class); String string = org.apache.commons.io.IOUtils.toString(inputStream, "UTF-8"); return string; } + + + } + + diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index a39310dd67..00e1122f09 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -99,9 +99,7 @@ public Response buildForwardResponseGET(String url, String mimetype) throws Prox WebResource.Builder b = buidFowrardResponse(url); ClientResponse response = b.get(ClientResponse.class); if (response.getStatus() == 200) { - String responseMimeType = response.getType().toString(); - InputStream is = response.getEntityInputStream(); MultivaluedMap headers = response.getHeaders(); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java index c1bda963a4..f4e2d9daa0 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java @@ -1,6 +1,7 @@ package cz.incad.kramerius.rest.apiNew.client.v60.redirection.item; import java.io.ByteArrayOutputStream; +import java.io.InputStream; import java.util.Date; import java.util.function.Consumer; import java.util.logging.Logger; @@ -71,6 +72,11 @@ public ProxyItemHandler(Instances instances, User user, Client client, SolrAcces public abstract Response audioMP3() throws ProxyHandlerException; public abstract Response audioOGG() throws ProxyHandlerException; public abstract Response audioWAV() throws ProxyHandlerException; + + + // helper methods for direct access to dc stream + public abstract InputStream directStreamDC() throws ProxyHandlerException; + public boolean imageThumbForceRedirection() { boolean redirection = KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + this.source + ".thumb", false); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java index 22752b17c6..ca3b50f7fe 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java @@ -1,6 +1,7 @@ package cz.incad.kramerius.rest.apiNew.client.v60.redirection.item; import java.io.IOException; +import java.io.InputStream; import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URISyntaxException; @@ -22,6 +23,7 @@ import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientHandlerException; +import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource; @@ -98,6 +100,20 @@ public Response dc(RequestMethodName method) throws ProxyHandlerException { return buildRedirectResponse(url); } + + + @Override + public InputStream directStreamDC() throws ProxyHandlerException { + String baseurl = super.baseUrl(); + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/item/" + this.pid + "/streams/DC"; + WebResource.Builder b = buidFowrardResponse(url); + ClientResponse response = b.get(ClientResponse.class); + if (response.getStatus() == 200) { + InputStream is = response.getEntityInputStream(); + return is; + } else return null; + } + @Override public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHandlerException { String baseurl = baseUrl(); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java index 2949932793..11994cd20f 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java @@ -1,5 +1,6 @@ package cz.incad.kramerius.rest.apiNew.client.v60.redirection.item; +import java.io.InputStream; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; @@ -8,6 +9,8 @@ import javax.ws.rs.core.Response; import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.SolrAccess; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; @@ -183,6 +186,21 @@ public Response audioWAV() throws ProxyHandlerException { } + + + @Override + public InputStream directStreamDC() throws ProxyHandlerException { + String baseurl = baseUrl(); + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/client/v7.0/items/" + this.pid + "/metadata/dc"; + WebResource.Builder b = buidFowrardResponse(url); + ClientResponse response = b.get(ClientResponse.class); + if (response.getStatus() == 200) { + InputStream is = response.getEntityInputStream(); + return is; + } else return null; + } + + @Override public Response iiifInfo(RequestMethodName method, String pid) throws ProxyHandlerException { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java index c9886a32d5..220f330842 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java @@ -17,24 +17,43 @@ package cz.incad.kramerius.rest.oai; import java.io.IOException; +import java.io.InputStream; import java.net.URL; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; +import javax.inject.Named; +import javax.inject.Provider; import javax.servlet.http.HttpServletRequest; +import javax.ws.rs.core.Response; +import javax.xml.parsers.ParserConfigurationException; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import com.google.inject.Inject; +import com.sun.jersey.api.client.Client; import cz.incad.kramerius.FedoraAccess; import cz.incad.kramerius.FedoraNamespaces; +import cz.incad.kramerius.SolrAccess; import cz.incad.kramerius.fedora.om.RepositoryException; +import cz.incad.kramerius.fedora.utils.CDKUtils; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; +import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; +import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.ProxyItemHandler; +import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.ProxyItemHandler.RequestMethodName; import cz.incad.kramerius.rest.oai.exceptions.OAIException; +import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.ApplicationURL; +import cz.incad.kramerius.utils.IPAddressUtils; import cz.incad.kramerius.utils.XMLUtils; import cz.incad.kramerius.utils.conf.KConfiguration; +import cz.incad.kramerius.utils.pid.LexerException; public enum MetadataExport { @@ -44,178 +63,210 @@ public enum MetadataExport { "http://www.openarchives.org/OAI/2.0/oai_dc/") { @Override - public Element perform(HttpServletRequest request, FedoraAccess fa, Document owningDocument, String oaiIdentifier,OAISet set) { - try { - String pid = OAITools.pidFromOAIIdentifier(oaiIdentifier); - Document dc = fa.getDC(pid); - if (dc != null) { - Element rootElement = dc.getDocumentElement(); - owningDocument.adoptNode(rootElement); - return rootElement; - } else return null; - } catch (IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - throw new RuntimeException(e.getMessage()); - } + public Element performOnCDKSide(SolrAccess solrAccess,Provider userProvider, Provider clientProvider, Instances instances, HttpServletRequest request, Document owningDocument, OAIRecord oaiRec,OAISet set) { + try { + String pid = OAITools.pidFromOAIIdentifier(oaiRec.getIdentifier()); + InputStream directStreamDC = dcStream(solrAccess, userProvider, clientProvider, instances, + request, pid); + if (directStreamDC != null) { + Document dc = XMLUtils.parseDocument(directStreamDC, true); + Element rootElement = dc.getDocumentElement(); + owningDocument.adoptNode(rootElement); + return rootElement; + } else { + return null; } + } catch (IOException | LexerException | ProxyHandlerException | ParserConfigurationException | SAXException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + throw new RuntimeException(e.getMessage()); + } + } }, edm("edm","http://www.europeana.eu/schemas/ese/","") { @Override - public Element perform(HttpServletRequest request, FedoraAccess fa, Document owningDocument, - String oaiIdentifier,OAISet set) { + public Element performOnCDKSide(SolrAccess solrAccess,Provider userProvider, Provider clientProvider, Instances instances, HttpServletRequest request, Document owningDocument, OAIRecord oaiRec,OAISet set) { try { String baseUrl = ApplicationURL.applicationURL(request); //rdf:about="uuid:6b182ad3-b9e9-11e1-1726-001143e3f55c" - String pid = OAITools.pidFromOAIIdentifier(oaiIdentifier); - Document dc = fa.getDC(pid); - Element dcElement = dc.getDocumentElement(); - - Element metadata = owningDocument.createElement("metadata"); - metadata.setAttribute("xmlns:europeana", "http://www.europeana.eu/schemas/ese/"); - metadata.setAttribute("xmlns:ore", "http://www.openarchives.org/ore/terms/"); - metadata.setAttribute("xmlns:edm", "http://www.europeana.eu/schemas/edm/"); - metadata.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); - metadata.setAttribute("xmlns:rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#"); - metadata.setAttribute("xmlns:rdaGr2", "http://rdvocab.info/ElementsGr2/"); - metadata.setAttribute("xmlns:skos", "http://www.w3.org/2004/02/skos/core#"); - metadata.setAttribute("xmlns:oai_dc", "http://www.openarchives.org/OAI/2.0/oai_dc/"); - metadata.setAttribute("xmlns:dc", "http://purl.org/dc/elements/1.1/"); - metadata.setAttribute("xmlns:dcterms", "http://purl.org/dc/terms/"); - - Element rdf = owningDocument.createElementNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#","rdf:RDF"); - rdf.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",oaiIdentifier); - metadata.appendChild(rdf); + String pid = OAITools.pidFromOAIIdentifier(oaiRec.getIdentifier()); + InputStream directStreamDC = dcStream(solrAccess, userProvider, clientProvider, instances, + request, pid); + if (directStreamDC != null) { + Document dc = XMLUtils.parseDocument(directStreamDC, true); + Element dcElement = dc.getDocumentElement(); + + Element metadata = owningDocument.createElement("metadata"); + metadata.setAttribute("xmlns:europeana", "http://www.europeana.eu/schemas/ese/"); + metadata.setAttribute("xmlns:ore", "http://www.openarchives.org/ore/terms/"); + metadata.setAttribute("xmlns:edm", "http://www.europeana.eu/schemas/edm/"); + metadata.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); + metadata.setAttribute("xmlns:rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#"); + metadata.setAttribute("xmlns:rdaGr2", "http://rdvocab.info/ElementsGr2/"); + metadata.setAttribute("xmlns:skos", "http://www.w3.org/2004/02/skos/core#"); + metadata.setAttribute("xmlns:oai_dc", "http://www.openarchives.org/OAI/2.0/oai_dc/"); + metadata.setAttribute("xmlns:dc", "http://purl.org/dc/elements/1.1/"); + metadata.setAttribute("xmlns:dcterms", "http://purl.org/dc/terms/"); + + Element rdf = owningDocument.createElementNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#","rdf:RDF"); + rdf.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",oaiRec.getIdentifier()); + metadata.appendChild(rdf); - Element providedCHO = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:ProvidedCHO"); - rdf.appendChild(providedCHO); - - List elements = XMLUtils.getElements(dcElement); - elements.stream().forEach(dcElm -> { - owningDocument.adoptNode(dcElm); - providedCHO.appendChild(dcElm); - }); - - //rdf.appendChild(providedCHO); - - Element type = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:type"); - providedCHO.appendChild(type); - type.setTextContent("TEXT"); - - Element webresource = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:WebResource"); - webresource.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",String.format("%s/api/client/v7.0/items/%s/image", baseUrl, pid)); - metadata.appendChild(webresource); - - Element edmAggregation = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:Aggregation"); - String clientUrl = KConfiguration.getInstance().getConfiguration().getString("client"); - if (clientUrl != null) { - edmAggregation.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); - } else { - edmAggregation.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); - } - Element edmDataPrvovider = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:dataProvider"); - - String edmDataProvider = set != null && set.getAdditionalsInfo() != null ? set.getAdditionalsInfo().get("edm:dataProvider") : null; - if (edmDataProvider != null) { - edmDataPrvovider.setTextContent(edmDataProvider); - } else { - edmDataPrvovider.setTextContent("Academy of Sciences Library/Knihovna Akademie věd ČR"); - } - edmAggregation.appendChild(edmDataPrvovider); - - // dodat dle setu - Element shownAt = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:isShownAt"); - if (clientUrl != null) { - shownAt.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource",clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); + Element providedCHO = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:ProvidedCHO"); + rdf.appendChild(providedCHO); + + List elements = XMLUtils.getElements(dcElement); + elements.stream().forEach(dcElm -> { + owningDocument.adoptNode(dcElm); + providedCHO.appendChild(dcElm); + }); + + //rdf.appendChild(providedCHO); + + Element type = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:type"); + providedCHO.appendChild(type); + type.setTextContent("TEXT"); + + Element webresource = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:WebResource"); + webresource.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",String.format("%s/api/client/v7.0/items/%s/image", baseUrl, pid)); + metadata.appendChild(webresource); + + Element edmAggregation = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:Aggregation"); + String clientUrl = KConfiguration.getInstance().getConfiguration().getString("client"); + if (clientUrl != null) { + edmAggregation.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); + } else { + edmAggregation.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); + } + Element edmDataPrvovider = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:dataProvider"); + + // Data provider + // find data provider by acronym + String acronym = KConfiguration.getInstance().getConfiguration().getString("acronym",""); + String defaultEDMDataProvider = KConfiguration.getInstance().getConfiguration().getString("oai.set.edm.dataProvider",acronym); +// if (edmDataProvider != null) { +// edmDataPrvovider.setTextContent(edmDataProvider); +// } + metadataProvider(instances, oaiRec, edmDataPrvovider, defaultEDMDataProvider); + edmAggregation.appendChild(edmDataPrvovider); + + // dodat dle setu + Element shownAt = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:isShownAt"); + if (clientUrl != null) { + shownAt.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource",clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); + } else { + shownAt.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource",baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); + } + edmAggregation.appendChild(shownAt); + + // mapovani na licence + Element edmRights = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:rights"); + + if (clientUrl != null) { + edmRights.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource", clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); + } else { + edmRights.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource", baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); + } + + + edmAggregation.appendChild(edmRights); + + + Element edmObject = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:object"); + edmObject.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource", String.format("%s/api/client/v7.0/items/%s/image", baseUrl, pid)); + edmAggregation.appendChild(edmObject); + + + // ceska digitalni kniovna + Element edmProvider = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:provider"); + String edmProviderText = KConfiguration.getInstance().getConfiguration().getString("oai.set.edm.provider",acronym); + edmProvider.setTextContent( edmProviderText); //"Czech digital library/Česká digitální knihovna"); + edmAggregation.appendChild(edmProvider); + + metadata.appendChild(edmAggregation); + + return metadata; } else { - shownAt.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource",baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); + return null; } - edmAggregation.appendChild(shownAt); - - // mapovani na licence - Element edmRights = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:rights"); - edmRights.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource", "https://cdk.lib.cas.cz/uuid/"+pid); - edmAggregation.appendChild(edmRights); - - - Element edmObject = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:object"); - edmObject.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource", String.format("%s/api/client/v7.0/items/%s/image", baseUrl, pid)); - edmAggregation.appendChild(edmObject); - - Element edmProvider = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:provider"); - edmProvider.setTextContent("Czech digital library/Česká digitální knihovna"); - edmAggregation.appendChild(edmProvider); - - metadata.appendChild(edmAggregation); - - return metadata; - } catch (IOException e) { + } catch (IOException | LexerException | ProxyHandlerException | ParserConfigurationException | SAXException e) { LOGGER.log(Level.SEVERE,e.getMessage(),e); throw new RuntimeException(e.getMessage()); } } - }; + }, - /** Disabled in CDK */ -// ese("ese", -// "http://www.europeana.eu/schemas/ese/ESE-V3.2.xsd", -// "http://www.europeana.eu/schemas/ese/") { -// @Override -// public Element perform(HttpServletRequest request, FedoraAccess fa, Document owningDocument, String oaiIdentifier,OAISet set) { -// try { -// String baseUrl = ApplicationURL.applicationURL(request); -// String pid = OAITools.pidFromOAIIdentifier(oaiIdentifier); -// Document dc = fa.getDC(pid); -// Element dcElement = dc.getDocumentElement(); -// -// Element record = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "record"); -// record.setAttribute("xmlns:dc", FedoraNamespaces.DC_NAMESPACE_URI); -// -// record.setAttributeNS("http://www.w3.org/2001/XMLSchema-instance","xsi:schemaLocation","http://www.europeana.eu/schemas/ese/ http://www.europeana.eu/schemas/ese/ESE-V3.2.xsd http://purl.org/dc/elements/1.1/ http://www.dublincore.org/schemas/xmls/qdc/dc.xsd http://purl.org/dc/terms/ http://www.dublincore.org/schemas/xmls/qdc/dcterms.xsd http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd"); -// List dcElems = XMLUtils.getElements(dcElement); -// dcElems.stream().forEach(dcElm-> { -// owningDocument.adoptNode(dcElm); -// record.appendChild(dcElm); -// }); -// -// Element object = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "object"); -// String thumb = String.format(baseUrl+(baseUrl.endsWith("/")? "" : "/")+"api/client/v7.0/items/%s/image/thumb", pid); -// object.setTextContent(thumb); -// record.appendChild(object); -// -// Element provider = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "provider"); -// provider.setTextContent("Academy of Sciences Library"); //TODO: To configuration -// record.appendChild(provider); -// -// Element type = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "type"); -// type.setTextContent("TEXT"); -// record.appendChild(type); -// -// Element isShownAt = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "isShownAt"); -// String clientUrl = KConfiguration.getInstance().getConfiguration().getString("client"); -// if (clientUrl != null) { -// isShownAt.setTextContent(clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); -// } else { -// isShownAt.setTextContent(baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); -// } -// record.appendChild(isShownAt); -// -// -// return record; -// -// } catch (IOException e) { -// LOGGER.log(Level.SEVERE,e.getMessage(),e); -// throw new RuntimeException(e.getMessage()); -// } -// } -// }; + ese("ese", + "http://www.europeana.eu/schemas/ese/ESE-V3.2.xsd", + "http://www.europeana.eu/schemas/ese/") { + + + + @Override + public Element performOnCDKSide(SolrAccess solrAccess,Provider userProvider, Provider clientProvider, Instances instances, HttpServletRequest request, Document owningDocument, OAIRecord oaiRec,OAISet set) { + try { + String baseUrl = ApplicationURL.applicationURL(request); + String pid = OAITools.pidFromOAIIdentifier(oaiRec.getIdentifier()); + + InputStream directStreamDC = dcStream(solrAccess, userProvider, clientProvider, instances, + request,pid); + if (directStreamDC != null) { + Document dc = XMLUtils.parseDocument(directStreamDC, true); + Element dcElement = dc.getDocumentElement(); + + Element record = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "record"); + record.setAttribute("xmlns:dc", FedoraNamespaces.DC_NAMESPACE_URI); + + record.setAttributeNS("http://www.w3.org/2001/XMLSchema-instance","xsi:schemaLocation","http://www.europeana.eu/schemas/ese/ http://www.europeana.eu/schemas/ese/ESE-V3.2.xsd http://purl.org/dc/elements/1.1/ http://www.dublincore.org/schemas/xmls/qdc/dc.xsd http://purl.org/dc/terms/ http://www.dublincore.org/schemas/xmls/qdc/dcterms.xsd http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd"); + List dcElems = XMLUtils.getElements(dcElement); + dcElems.stream().forEach(dcElm-> { + owningDocument.adoptNode(dcElm); + record.appendChild(dcElm); + }); + + Element object = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "object"); + String thumb = String.format(baseUrl+(baseUrl.endsWith("/")? "" : "/")+"api/client/v7.0/items/%s/image/thumb", pid); + object.setTextContent(thumb); + record.appendChild(object); + + Element provider = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "provider"); + String acronym = KConfiguration.getInstance().getConfiguration().getString("acronym",""); + String defaultDataProvider = KConfiguration.getInstance().getConfiguration().getString("oai.set.edm.dataProvider",acronym); + + metadataProvider(instances, oaiRec, provider, defaultDataProvider); + + record.appendChild(provider); + + Element type = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "type"); + type.setTextContent("TEXT"); + record.appendChild(type); + + Element isShownAt = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "isShownAt"); + String clientUrl = KConfiguration.getInstance().getConfiguration().getString("client"); + if (clientUrl != null) { + isShownAt.setTextContent(clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); + } else { + isShownAt.setTextContent(baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); + } + record.appendChild(isShownAt); + + + return record; + } else return null; + + } catch (IOException | LexerException | ProxyHandlerException | ParserConfigurationException | SAXException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + throw new RuntimeException(e.getMessage()); + } + } + }; /** disabled for now */ // drkramerius4("drkramerius4", @@ -228,6 +279,7 @@ public Element perform(HttpServletRequest request, FedoraAccess fa, Document own // } // }; + public static final Logger LOGGER = Logger.getLogger(MetadataExport.class.getName()); private MetadataExport(String metadataPrefix, String schema, String metadataNamespace) { @@ -257,15 +309,81 @@ public static MetadataExport findByPrefix(String prefix) { return null; } + private static InputStream dcStream(SolrAccess solrAccess, Provider userProvider, + Provider clientProvider, Instances instances, HttpServletRequest request, String pid) + throws LexerException, IOException, ProxyHandlerException { + ProxyItemHandler redirectHandler = findRedirectHandler(solrAccess, userProvider, clientProvider, instances, request, pid, null); + InputStream directStreamDC = redirectHandler.directStreamDC(); + return directStreamDC; + } + + public Document getDC(ProxyItemHandler handler) throws ProxyHandlerException, ParserConfigurationException, SAXException, IOException { + InputStream is = handler.directStreamDC(); + if (is != null) { + Document document = XMLUtils.parseDocument(is, true); + return document; + } else return null; + + } + private String metadataPrefix; private String schema; private String metadataNamespace; + /* + * @Inject + @Named("forward-client") + Provider clientProvider; + + */ - public abstract Element perform(HttpServletRequest request, FedoraAccess fa, Document owningDocument, String oaiIdentifier, OAISet set); + public static ProxyItemHandler findRedirectHandler(SolrAccess solrAccess,Provider userProvider, Provider clientProvider, Instances instances, HttpServletRequest request, String pid, String source) throws LexerException, IOException { + if (source == null) { + source = defaultDocumentSource(solrAccess, pid); + } + OneInstance found = instances.find(source); + if (found!= null) { + String remoteAddress = IPAddressUtils.getRemoteAddress(request, KConfiguration.getInstance().getConfiguration()); + ProxyItemHandler proxyHandler = found.createProxyItemHandler(userProvider.get(), clientProvider.get(), solrAccess, source, pid, remoteAddress); + return proxyHandler; + } else { + return null; + } + } + + private static String defaultDocumentSource(SolrAccess solrAccess, String pid) throws IOException { + org.w3c.dom.Document solrDataByPid = solrAccess.getSolrDataByPid(pid); + // TODO: All page + String leader = CDKUtils.findCDKLeader(solrDataByPid.getDocumentElement()); + List sources = CDKUtils.findSources(solrDataByPid.getDocumentElement()); + return leader != null ? leader : (!sources.isEmpty() ? sources.get(0) : null); + } + - public static final Logger LOGGER = Logger.getLogger(MetadataExport.class.getName()); + + private static void metadataProvider(Instances instances, OAIRecord oaiRec, Element provider, String defaultValue) { + List cdkCollections = oaiRec.getCdkCollections(); + if (cdkCollections.size() > 0) { + String acronym = cdkCollections.get(0); + OneInstance found = instances.find(acronym.trim()); + if (found != null) { + provider.setTextContent(found.getRegistrInfo().get(OneInstance.NAME_ENG)); + } else { + provider.setTextContent(acronym.trim()); + } + + } else { + provider.setTextContent(defaultValue); + } + } + + public abstract Element performOnCDKSide(SolrAccess solrAccess,Provider userProvider, Provider clientProvider, Instances instances, HttpServletRequest request, Document owningDocument, OAIRecord oaiRec, OAISet set); + + //public abstract Element perform(HttpServletRequest request, ProxyItemHandler handler, Document owningDocument, String oaiIdentifier, OAISet set); + /* + * ProxyItemHandler handler, + */ // public static final Logger LOGGER = Logger.getLogger(MetadataExport.class.getName()); } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIEndpoint.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIEndpoint.java index 1416a7d61c..0252474697 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIEndpoint.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIEndpoint.java @@ -31,49 +31,114 @@ import javax.ws.rs.core.PathSegment; import javax.ws.rs.core.Response; import javax.ws.rs.core.UriInfo; +import javax.xml.parsers.ParserConfigurationException; import org.json.JSONObject; import org.w3c.dom.Document; import org.w3c.dom.Element; +import org.xml.sax.SAXException; import com.google.inject.Provider; +import com.sun.jersey.api.client.Client; import cz.incad.kramerius.FedoraAccess; import cz.incad.kramerius.SolrAccess; import cz.incad.kramerius.repository.KrameriusRepositoryApiImpl; -import cz.incad.kramerius.rest.apiNew.client.v60.ClientApiResource; +import cz.incad.kramerius.rest.apiNew.ConfigManager; +import cz.incad.kramerius.rest.apiNew.client.v60.filter.ProxyFilter; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.exceptions.InternalErrorException; import cz.incad.kramerius.rest.oai.exceptions.OAIException; +import cz.incad.kramerius.rest.oai.exceptions.OAIInfoException; +import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.ApplicationURL; import cz.incad.kramerius.utils.StringUtils; import cz.incad.kramerius.utils.XMLUtils; import cz.incad.kramerius.utils.conf.KConfiguration; -import cz.incad.kramerius.SolrAccess; - - import static cz.incad.kramerius.rest.oai.OAITools.*; - +import java.io.IOException; import java.io.StringWriter; import java.util.logging.Level; import java.util.logging.Logger; @Path("/harvest/v7.0") -public class OAIEndpoint extends ClientApiResource { +public class OAIEndpoint extends cz.incad.kramerius.rest.apiNew.client.v60.ClientApiResource { public static Logger LOGGER = Logger.getLogger(OAIEndpoint.class.getName()); - @Inject - @Named("cachedFedoraAccess") - private transient FedoraAccess fedoraAccess; +// @Inject +// @Named("cachedFedoraAccess") +// private transient FedoraAccess fedoraAccess; @Inject Provider requestProvider; + @Inject + ConfigManager configManager; + + @Inject + @Named("forward-client") + Provider clientProvider; + + @Inject + Provider userProvider; + + @Inject + Instances instances; + + @Inject + ProxyFilter proxyFilter; + + public OAIEndpoint() { } + + @GET + @Path("info") + @Produces(MediaType.APPLICATION_JSON) + public Response info( + @QueryParam("set") String set + ) throws OAIException { + + +// private String host; +// private String setSpec; +// private String setName; +// private String setDescription; +// private String filterQuery; + + + OAISets sets = new OAISets(configManager,null); + OAISet found = sets.findBySet(set); + if (found != null) { + try { + int ndocs = found.numberOfDoc(proxyFilter, solrAccess); + JSONObject object = new JSONObject(); + object.put("setSpec", found.getSetSpec()); + object.put("setName", found.getSetName()); + object.put("setDescription", found.getSetDescription()); + object.put("filterQuery", found.getFilterQuery()); + object.put("numberDocs", ndocs); + + return Response.ok(object.toString()).type(MediaType.APPLICATION_JSON.toString()).build(); + } catch (IOException e) { + throw new OAIInfoException(e.getMessage()); + } catch (ParserConfigurationException e) { + throw new OAIInfoException(e.getMessage()); + } catch (SAXException e) { + throw new OAIInfoException(e.getMessage()); + } + } else { + if (set != null) { + throw new OAIInfoException(String.format("Set %s not found", set)); + } else { + throw new OAIInfoException("No default set found "); + } + } + } @GET @@ -90,7 +155,8 @@ public Response oai( OAIVerb oaiVerb = OAIVerb.valueOf(verb); Document oai = createOAIDocument(); Element oaiRoot = oai.getDocumentElement(); - oaiVerb.perform(this.fedoraAccess, this.getSolrAccess(), this.requestProvider.get(), oai, oaiRoot); + + oaiVerb.perform(this.userProvider, clientProvider, instances, configManager, this.proxyFilter, this.solrAccess, this.requestProvider.get(), oai, oaiRoot); StringWriter writer = new StringWriter(); XMLUtils.print(oai, writer); return Response.ok(writer.toString()).build(); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java index 14c46b65a8..b076164aa9 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java @@ -25,6 +25,7 @@ import java.util.List; import java.util.stream.Collectors; +import javax.inject.Provider; import javax.servlet.http.HttpServletRequest; import javax.xml.parsers.ParserConfigurationException; @@ -32,16 +33,27 @@ import org.w3c.dom.Element; import org.xml.sax.SAXException; +import com.sun.jersey.api.client.Client; + import cz.incad.kramerius.FedoraAccess; import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; +import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.ProxyItemHandler; +import cz.incad.kramerius.security.User; +import cz.incad.kramerius.utils.IPAddressUtils; import cz.incad.kramerius.utils.XMLUtils; +import cz.incad.kramerius.utils.conf.KConfiguration; +import cz.incad.kramerius.utils.pid.LexerException; public class OAIRecord { private String identifier; private String solrIdentifier; -// private List possibleSets = new ArrayList<>(); + // CDK extension + + private List cdkCollections = new ArrayList<>(); public OAIRecord(String solrIdentifier, String identifier) { super(); @@ -56,15 +68,16 @@ public String getIdentifier() { public String getSolrIdentifier() { return solrIdentifier; } - -// public List getPossibleSets() { -// return possibleSets; -// } -// -// public void setPossibleSets(List possibleSets) { -// this.possibleSets = possibleSets; -// } + + public List getCdkCollections() { + return cdkCollections; + } + + public void setCdkCollections(List cdkCollections) { + this.cdkCollections = cdkCollections; + } + /** find oai record */ public static OAIRecord findRecord(SolrAccess solrAccess,String oaiIdentifier) throws IOException, ParserConfigurationException, SAXException { String pid = OAITools.pidFromOAIIdentifier(oaiIdentifier); @@ -90,8 +103,29 @@ public boolean acceptElement(Element element) { }); if (docs.size() > 0) { - Element pidElm = XMLUtils.findElement(docs.get(0), "str"); - return new OAIRecord(pidElm.getTextContent(), oaiIdentifier); + Element doc = docs.get(0); + + Element pidElm = XMLUtils.findElement(doc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + return name.equals("pid"); + + } + }); + + List collections = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + return name.equals("cdk.collection"); + + } + }); + + OAIRecord oaiRecord = new OAIRecord(pidElm.getTextContent(), oaiIdentifier); + oaiRecord.setCdkCollections(collections.stream().map(Element::getTextContent).collect(Collectors.toList())); + return oaiRecord; } else return null; } else { @@ -101,10 +135,10 @@ public boolean acceptElement(Element element) { } /** render metadata */ - public Element toMetadata(HttpServletRequest request, FedoraAccess fa, Document doc, MetadataExport export, OAISet set) { - return export.perform(request, fa, doc, identifier, set); + public Element toMetadata(SolrAccess solrAccess,Provider userProvider, Provider clientProvider, Instances instances, HttpServletRequest request, Document owningDocument, String oaiIdentifier, MetadataExport export, OAISet set) { + return export.performOnCDKSide(solrAccess,userProvider, clientProvider, instances, request, owningDocument, this, set); } - +// /** render header */ public Element toHeader(Document doc, String setSpec) { Element header = doc.createElement("header"); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIResults.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIResults.java index 445e51987a..2abdc7fd59 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIResults.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIResults.java @@ -33,7 +33,6 @@ public OAIResults(int completeListSize, String resumptionToken, String metadataP this.metadataPrerix = metadataPrefix; } - public int getCompleteListSize() { return completeListSize; } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java index 3912650cd9..d455e0fdc9 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java @@ -33,6 +33,8 @@ import org.xml.sax.SAXException; import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.rest.apiNew.ConfigManager; +import cz.incad.kramerius.rest.apiNew.client.v60.filter.ProxyFilter; import cz.incad.kramerius.solr.SolrFieldsMapping; import cz.incad.kramerius.utils.ApplicationURL; import cz.incad.kramerius.utils.StringUtils; @@ -40,8 +42,9 @@ public class OAISet { - private String host; + public static final String DEFAULT_SET_KEYWORD = "DEFAULT"; + private String host; private String setSpec; private String setName; private String setDescription; @@ -73,8 +76,6 @@ public OAISet( } - - protected OAISet(String host) { super(); this.host= host; @@ -123,15 +124,58 @@ public Map getAdditionalsInfo() { return additionalsInfo; } + public void storeInitConfig(ConfigManager confMapanger) { + } + public boolean isMyResumptionToken(String resumptionToken) { String spec = OAITools.specFromResumptionToken(resumptionToken); return spec.equals(getSetSpec()); } - - public OAIResults findRecords(SolrAccess solrAccess,String cursor, String metadataPrefix, int rows) throws IOException, ParserConfigurationException, SAXException { + public void initToConfig(ConfigManager configManager) { - String query = String.format("q=%s&cursorMark=%s&fl=pid&rows=%d&sort=pid+asc", this.filterQuery, cursor, rows); + String filter = String.format("oai.set.%s.filter", this.setSpec); + String name = String.format("oai.set.%s.name", this.setSpec); + String desc = String.format("oai.set.%s.description", this.setSpec); + + String property = configManager.getProperty(filter); + if (property == null) { + configManager.setProperty(filter, this.getFilterQuery()); + configManager.setProperty(name, this.getSetName()); + configManager.setProperty(desc, this.getSetDescription()); + } + } + + + public int numberOfDoc(ProxyFilter proxyFilter, SolrAccess solrAccess) throws IOException, ParserConfigurationException, SAXException { + String query = String.format("q=%s&fl=pid&rows=%d&sort=pid+asc", this.filterQuery, 0); + if (proxyFilter.newFilter() != null) { + query = query + String.format("&fq=%s", proxyFilter.newFilter()); + } + String solrResponseXml = solrAccess.requestWithSelectReturningString(query, "xml"); + Document document = XMLUtils.parseDocument(new StringReader(solrResponseXml)); + Element result = XMLUtils.findElement(document.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + return element.getNodeName().equals("result"); + } + }); + + if (result != null) { + String number = result.getAttribute("numFound"); + return Integer.parseInt(number); + } + + return -1; + } + + public OAIResults findRecords(ProxyFilter proxyFilter, SolrAccess solrAccess,String cursor, String metadataPrefix, int rows) throws IOException, ParserConfigurationException, SAXException { + String fq = proxyFilter.newFilter(); + String query = String.format("q=%s&cursorMark=%s&fl=pid+cdk.leaders+cdk.collection+cdk.leader&rows=%d&sort=compositeId+asc", this.filterQuery, cursor, rows); + if (fq != null) { + query = query + String.format("&fq=%s", fq); + } + String solrResponseXml = solrAccess.requestWithSelectReturningString(query, "xml"); Document document = XMLUtils.parseDocument(new StringReader(solrResponseXml)); @@ -165,13 +209,35 @@ public boolean acceptElement(Element element) { }); List records = docs.stream().map(doc-> { - Element pidElm = XMLUtils.findElement(doc, "str"); + + Element pidElm = XMLUtils.findElement(doc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + return name.equals("pid"); + + } + }); + + + List collections = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + return name.equals("cdk.collection"); + + } + }); + List cdkCollections = collections.stream().map(Element::getTextContent).collect(Collectors.toList()); String oaiIdentifier = OAITools.oaiIdentfier(host, pidElm.getTextContent()); - return new OAIRecord(pidElm.getTextContent(), oaiIdentifier); + OAIRecord oaiRecord = new OAIRecord(pidElm.getTextContent(), oaiIdentifier); + oaiRecord.setCdkCollections(cdkCollections); + return oaiRecord; }).collect(Collectors.toList()); String nextCursor = records.size() == rows && solrNextCursor != null ? solrNextCursor+":"+this.setSpec+":"+metadataPrefix : null; OAIResults results = new OAIResults(Integer.parseInt(number), nextCursor, metadataPrefix, records); + return results; } else { return null; diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISets.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISets.java index 9b5394ae05..ce508419a1 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISets.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISets.java @@ -25,16 +25,29 @@ import java.util.Optional; import java.util.logging.Logger; +import cz.incad.kramerius.rest.apiNew.ConfigManager; import cz.incad.kramerius.utils.conf.KConfiguration; public class OAISets { + public static final String DEFAULT_SET_KEY = "DEFAULT"; + + private OAISet defaultSet; private List sets = new ArrayList<>(); public OAISets(String host) { + loadFromStandardConfiguration(host); + } + + public OAISets(ConfigManager manager, String host) { + loadFromConfigurationManager(manager, host); + } - /* Default sets are disabled + + + /* + private void embeddedSets(String host) { sets.addAll( Arrays.asList( @@ -78,65 +91,136 @@ public OAISets(String host) { "Set of periodical supplements", "-", "model:supplement") )); - */ - - - this.defaultSet = this.sets.get(0); - // configuration for new sets + }*/ + + private void loadFromConfigurationManager(ConfigManager confManager, String host) { + Map configuredSets = new HashMap(); - Iterator keys = KConfiguration.getInstance().getConfiguration().getKeys("oai.set"); - while(keys.hasNext()) { - String key = keys.next(); + List keys = confManager.getKeysByRegularExpression("^oai\\.set.*"); + for (String key : keys) { String rest = key.substring("oai.set.".length()); String[] values = rest.split("\\."); if (values.length == 2) { String spec = values[0]; - if (!configuredSets.containsKey(spec)) { - configuredSets.put(spec, new OAISet(host)); - configuredSets.get(spec).setSetSpec(spec); + OAISet pSet = null; + if (spec.equals(DEFAULT_SET_KEY)) { + if(this.defaultSet == null) { + this.defaultSet = new OAISet(host); + this.defaultSet.setSetSpec(spec); + } + pSet = this.defaultSet; + } else { + if (!configuredSets.containsKey(spec)) { + configuredSets.put(spec, new OAISet(host)); + configuredSets.get(spec).setSetSpec(spec); + } + pSet = configuredSets.get(spec); } + String property = values[1]; switch(property) { case "name":{ - OAISet set = configuredSets.get(spec); - set.setSetName(KConfiguration.getInstance().getProperty(key)); + //OAISet set = configuredSets.get(spec); + pSet.setSetName( confManager.getProperty(key)); } break; case "desc": case "description":{ - OAISet set = configuredSets.get(spec); - set.setSetDescription(KConfiguration.getInstance().getProperty(key)); + pSet.setSetDescription(confManager.getProperty(key)); } break; case "filter":{ - OAISet set = configuredSets.get(spec); - set.setFilterQuery(KConfiguration.getInstance().getProperty(key)); + pSet.setFilterQuery(confManager.getProperty(key)); } break; default: { - OAISet set = configuredSets.get(spec); - set.getAdditionalsInfo().put(key, KConfiguration.getInstance().getProperty(key)); + pSet.getAdditionalsInfo().put(key, confManager.getProperty(key)); } break; } } } configuredSets.values().stream().filter(oai -> oai.getFilterQuery() != null).forEach(sets::add); - LOGGER.info("OAI -> Configured sets :"+this.sets); + } + + private void loadFromStandardConfiguration(String host) { + Map configuredSets = new HashMap(); + Iterator keys = KConfiguration.getInstance().getConfiguration().getKeys("oai.set"); + while(keys.hasNext()) { + String key = keys.next(); + String rest = key.substring("oai.set.".length()); + String[] values = rest.split("\\."); + if (values.length == 2) { + String spec = values[0]; + + if (!spec.equals(DEFAULT_SET_KEY)) { + if (this.defaultSet == null) { + this.defaultSet = new OAISet(host); + } + loadOAISetPropertiesFromConf(key, values, this.defaultSet); + } else { + if (!configuredSets.containsKey(spec)) { + configuredSets.put(spec, new OAISet(host)); + configuredSets.get(spec).setSetSpec(spec); + } + OAISet pset = configuredSets.get(spec); + loadOAISetPropertiesFromConf(key, values, pset); + } + } + } + + configuredSets.values().stream().filter(oai -> oai.getFilterQuery() != null).forEach(sets::add); + //LOGGER.info("OAI -> Configured sets"); + } + + private void loadOAISetPropertiesFromConf(String key, String[] values, OAISet pset) { + String property = values[1]; + switch(property) { + case "name":{ + //OAISet set = configuredSets.get(spec); + pset.setSetName(KConfiguration.getInstance().getProperty(key)); + } + break; + case "desc": + case "description":{ + //OAISet set = configuredSets.get(spec); + pset.setSetDescription(KConfiguration.getInstance().getProperty(key)); + } + break; + case "filter":{ + //OAISet set = configuredSets.get(spec); + pset.setFilterQuery(KConfiguration.getInstance().getProperty(key)); + } + break; + default: { + //OAISet set = configuredSets.get(spec); + pset.getAdditionalsInfo().put(key, KConfiguration.getInstance().getProperty(key)); + } + break; + } } public OAISet findBySet(String setName) { - Optional found = this.sets.stream() - .filter(oaiSet -> oaiSet.getSetSpec().equals(setName)) - .findFirst(); - return found.isPresent() ? found.get() : null; + if (setName != null && !setName.equals(DEFAULT_SET_KEY)) { + Optional found = this.sets.stream() + .filter(oaiSet -> oaiSet.getSetSpec().equals(setName)) + .findFirst(); + return found.isPresent() ? found.get() : null; + } else { + return this.defaultSet; + } } public OAISet findByToken(String token) { - Optional found = this.sets.stream() - .filter(oaiSet -> oaiSet.isMyResumptionToken(token)) - .findFirst(); - return found.get(); + if (this.defaultSet.isMyResumptionToken(token)) { + return this.defaultSet; + } else { + Optional found = this.sets.stream() + .filter(oaiSet -> oaiSet.isMyResumptionToken(token)) + .findFirst(); + + return found.get(); + } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java index 5716a15214..ec49699e10 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java @@ -23,6 +23,7 @@ import java.util.logging.Level; import java.util.logging.Logger; +import javax.inject.Provider; import javax.servlet.http.HttpServletRequest; import javax.xml.parsers.ParserConfigurationException; @@ -31,9 +32,15 @@ import org.w3c.dom.Element; import org.xml.sax.SAXException; +import com.sun.jersey.api.client.Client; + import cz.incad.kramerius.FedoraAccess; import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.rest.apiNew.ConfigManager; +import cz.incad.kramerius.rest.apiNew.client.v60.filter.ProxyFilter; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.oai.exceptions.OAIException; +import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.ApplicationURL; import cz.incad.kramerius.utils.XMLUtils; import cz.incad.kramerius.utils.conf.KConfiguration; @@ -42,7 +49,7 @@ public enum OAIVerb { // metadata formats ListMetadataFormats { @Override - public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException{ + public void perform(Provider userProvider, Provider clientProvider, Instances instances, ConfigManager configManager,ProxyFilter proxyFilter, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException{ Element requestElement = OAITools.requestElement(doc, OAIVerb.ListMetadataFormats,null,ApplicationURL.applicationURL(request),null); doc.getDocumentElement().appendChild(requestElement); @@ -74,7 +81,7 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r }, ListSets { @Override - public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException{ + public void perform(Provider userProvider,Provider clientProvider, Instances instances, ConfigManager configManager,ProxyFilter proxyFilter, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException{ try { Element requestElement = OAITools.requestElement(doc, OAIVerb.ListSets, null,ApplicationURL.applicationURL(request), null); doc.getDocumentElement().appendChild(requestElement); @@ -82,30 +89,32 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r String baseUrl = ApplicationURL.applicationURL(request); URL urlObject = new URL(baseUrl); - OAISets sets = new OAISets(urlObject.getHost()); + OAISets sets = new OAISets(configManager, urlObject.getHost()); Element listSets = doc.createElement("ListSets"); doc.getDocumentElement().appendChild(listSets); List oaiSets = sets.getAOISets(); for (OAISet oaiIterationSet : oaiSets) { - Element setDefinition= doc.createElement("set"); + if (!oaiIterationSet.getSetSpec().equals(OAISet.DEFAULT_SET_KEYWORD)) { + Element setDefinition= doc.createElement("set"); - Element setSpec = doc.createElement("setSpec"); - setSpec.setTextContent(oaiIterationSet.getSetSpec()); - setDefinition.appendChild(setSpec); + Element setSpec = doc.createElement("setSpec"); + setSpec.setTextContent(oaiIterationSet.getSetSpec()); + setDefinition.appendChild(setSpec); - Element setName = doc.createElement("setName"); - setName.setTextContent(oaiIterationSet.getSetName()); - setDefinition.appendChild(setName); - - Element setDescription = doc.createElement("setDescription"); - if (oaiIterationSet.getSetDescription() != null) { - setDescription.setTextContent(oaiIterationSet.getSetDescription()); + Element setName = doc.createElement("setName"); + setName.setTextContent(oaiIterationSet.getSetName()); + setDefinition.appendChild(setName); + + Element setDescription = doc.createElement("setDescription"); + if (oaiIterationSet.getSetDescription() != null) { + setDescription.setTextContent(oaiIterationSet.getSetDescription()); + } + setDefinition.appendChild(setDescription); + + listSets.appendChild(setDefinition); } - setDefinition.appendChild(setDescription); - - listSets.appendChild(setDefinition); } doc.getDocumentElement().appendChild(listSets); @@ -119,7 +128,7 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r Identify { @Override - public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException { + public void perform(Provider userProvider,Provider clientProvider, Instances instances, ConfigManager configManager,ProxyFilter proxyFilter, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException { try { String url = ApplicationURL.applicationURL(request); @@ -203,17 +212,16 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r ListRecords { @Override - public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException{ + public void perform(Provider userProvider,Provider clientProvider, Instances instances, ConfigManager configManager,ProxyFilter proxyFilter, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException{ OAISet selectedSet = null; MetadataExport selectedMetadata = null; try { String baseUrl = ApplicationURL.applicationURL(request); - - URL urlObject = new URL(baseUrl); - OAISets sets = new OAISets(urlObject.getHost()); + OAISets sets = new OAISets(configManager, urlObject.getHost()); String set = request.getParameter("set"); + String resumptionToken = request.getParameter("resumptionToken"); String metadataPrefix = request.getParameter("metadataPrefix"); if (metadataPrefix != null || resumptionToken != null) { @@ -222,12 +230,11 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r selectedSet = sets.findBySet(set); if (selectedSet ==null) { throw new OAIException(ErrorCode.badArgument, OAIVerb.ListRecords, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); - } } else if (resumptionToken != null){ selectedSet = sets.findByToken(resumptionToken); metadataPrefix = OAITools.metadataFromResumptionToken(resumptionToken); - if (selectedSet == null || metadataPrefix == null || MetadataExport.findByPrefix(metadataPrefix) == null) { + if (metadataPrefix == null || MetadataExport.findByPrefix(metadataPrefix) == null) { throw new OAIException(ErrorCode.badResumptionToken, OAIVerb.ListRecords, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); } } @@ -247,15 +254,16 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r if (resumptionToken != null) { String solrCursor = OAITools.solrCursorMarkFromResumptionToken(resumptionToken); - results = selectedSet.findRecords(solrAccess, solrCursor,metadataPrefix,rows); + results = selectedSet.findRecords(proxyFilter, solrAccess, solrCursor,metadataPrefix,rows); for (OAIRecord oaiRec : results.getRecords()) { Element record= doc.createElement("record"); Element header = oaiRec.toHeader(doc, selectedSet.getSetSpec()); + // Instances instances, HttpServletRequest request, Document owningDocument, String oaiIdentifier, MetadataExport export, OAISet set Element metadata = doc.createElement("metadata"); - metadata.appendChild(oaiRec.toMetadata(request, fa, doc, selectedMetadata, selectedSet)); + metadata.appendChild(oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,selectedSet)); record.appendChild(header); record.appendChild(metadata); @@ -265,7 +273,7 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r } } else { - results = selectedSet.findRecords(solrAccess,"*", metadataPrefix,rows); + results = selectedSet.findRecords(proxyFilter, solrAccess,"*", metadataPrefix,rows); for (OAIRecord oaiRec : results.getRecords()) { Element record= doc.createElement("record"); @@ -273,7 +281,7 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r Element metadata = doc.createElement("metadata"); - metadata.appendChild(oaiRec.toMetadata(request, fa, doc, selectedMetadata, selectedSet)); + metadata.appendChild(oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,selectedSet)); record.appendChild(header); record.appendChild(metadata); @@ -306,7 +314,7 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r }, ListIdentifiers { @Override - public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException{ + public void perform(Provider userProvider,Provider clientProvider, Instances instances, ConfigManager configManager,ProxyFilter proxyFilter, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException{ OAISet selectedSet = null; MetadataExport selectedMetadata = null; @@ -315,19 +323,19 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r URL urlObject = new URL(baseUrl); - OAISets sets = new OAISets(urlObject.getHost()); + OAISets sets = new OAISets(configManager, urlObject.getHost()); String set = request.getParameter("set"); String resumptionToken = request.getParameter("resumptionToken"); String metadataPrefix = request.getParameter("metadataPrefix"); if (metadataPrefix != null || resumptionToken != null) { int rows = KConfiguration.getInstance().getConfiguration().getInt(REPOSITORY_ROWS_IN_RESULTS,600); - + if (set != null) { selectedSet = sets.findBySet(set); } else if (resumptionToken != null){ selectedSet = sets.findByToken(resumptionToken); metadataPrefix = OAITools.metadataFromResumptionToken(resumptionToken); - if (selectedSet == null || metadataPrefix == null || MetadataExport.findByPrefix(metadataPrefix) == null) { + if ( metadataPrefix == null || MetadataExport.findByPrefix(metadataPrefix) == null) { throw new OAIException(ErrorCode.badResumptionToken, OAIVerb.ListIdentifiers, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); } } @@ -346,10 +354,10 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r OAIResults results = null; if (resumptionToken != null) { String solrCursor = OAITools.solrCursorMarkFromResumptionToken(resumptionToken); - results = selectedSet.findRecords(solrAccess, solrCursor,metadataPrefix,rows); + results = selectedSet.findRecords(proxyFilter, solrAccess, solrCursor,metadataPrefix,rows); for (OAIRecord oaiRec : results.getRecords()) { identify.appendChild(oaiRec.toHeader(doc, selectedSet.getSetSpec()));} } else { - results = selectedSet.findRecords(solrAccess,"*", metadataPrefix,rows); + results = selectedSet.findRecords(proxyFilter, solrAccess,"*", metadataPrefix,rows); for (OAIRecord oaiRec : results.getRecords()) { identify.appendChild(oaiRec.toHeader(doc, selectedSet.getSetSpec()));} } @@ -377,7 +385,7 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r }, GetRecord { @Override - public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException{ + public void perform(Provider userProvider,Provider clientProvider, Instances instances, ConfigManager configManager, ProxyFilter proxyFilter, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws OAIException{ MetadataExport selectedMetadata = null; try { @@ -405,7 +413,7 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r Element metadata = doc.createElement("metadata"); - metadata.appendChild(oaiRec.toMetadata(request, fa, doc, selectedMetadata, null)); + metadata.appendChild(oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,null)); record.appendChild(header); record.appendChild(metadata); @@ -433,7 +441,9 @@ public void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest r private static final String REPOSITORY_NAME = "oai.repositoryName"; private static final String REPOSITORY_ROWS_IN_RESULTS = "oai.rowsInResults"; - public abstract void perform(FedoraAccess fa, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws Exception; + + // PERFORM + public abstract void perform( Provider userPRovider, Provider clientProvider, Instances instances, ConfigManager configManager, ProxyFilter proxyFilter, SolrAccess solrAccess, HttpServletRequest request, Document doc, Element rootElement) throws Exception; public static Logger LOGGER = Logger.getLogger(OAIVerb.class.getName()); } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/db/OAIDBInitializer.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/db/OAIDBInitializer.java new file mode 100644 index 0000000000..632ce079cd --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/db/OAIDBInitializer.java @@ -0,0 +1,67 @@ +/* + * Copyright (C) Feb 5, 2024 Pavel Stastny + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cz.incad.kramerius.rest.oai.db; + +import static cz.incad.kramerius.database.cond.ConditionsInterpretHelper.versionCondition; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.Charset; +import java.sql.Connection; +import java.sql.SQLException; +import java.util.logging.Level; +import java.util.logging.Logger; + +import cz.incad.kramerius.database.VersionService; +import cz.incad.kramerius.utils.DatabaseUtils; +import cz.incad.kramerius.utils.IOUtils; +import cz.incad.kramerius.utils.database.JDBCUpdateTemplate; + +public class OAIDBInitializer { + + static Logger LOGGER = Logger.getLogger(OAIDBInitializer.class.getName()); + + public static void initDatabase(Connection connection, VersionService versionService) { + try { + String version = versionService.getVersion(); + if (version == null ) { + initDefaultOAISets(connection); + } else if (versionCondition(version, "<=", "7.0.3")){ + initDefaultOAISets(connection); + } + + } catch (SQLException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } + } + + private static void initDefaultOAISets(Connection connection) { + try { + InputStream is = OAIDBInitializer.class.getClassLoader().getResourceAsStream("/initoaidb.sql"); + JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); + template.setUseReturningKeys(false); + String sqlScript = IOUtils.readAsString(is, Charset.forName("UTF-8"), true); + template.executeUpdate(sqlScript); + } catch (IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } catch (SQLException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } + + } + +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/exceptions/OAIInfoException.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/exceptions/OAIInfoException.java new file mode 100644 index 0000000000..ebe161b1d2 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/exceptions/OAIInfoException.java @@ -0,0 +1,61 @@ +/* + * Copyright (C) Feb 3, 2024 Pavel Stastny + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cz.incad.kramerius.rest.oai.exceptions; + +import java.util.logging.Logger; + +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.json.JSONObject; + +import cz.incad.kramerius.rest.oai.ErrorCode; +import cz.incad.kramerius.rest.oai.MetadataExport; +import cz.incad.kramerius.rest.oai.OAISet; +import cz.incad.kramerius.rest.oai.OAIVerb; + +public class OAIInfoException extends WebApplicationException { + public static final Logger LOGGER = Logger.getLogger(OAIException.class.getName()); + + protected OAIInfoException(Response response) { + super(response); + } + +// public AbstractOAIException(int errorCode) { +// super(errorCode); +// } +// +// public AbstractOAIException(Response.Status status) { +// super(status); +// } + + public OAIInfoException(String errMessage) { + this(Response.status(Response.Status.BAD_REQUEST) + .type(MediaType.APPLICATION_JSON) + .entity(buildJSON(errMessage)) + .build()); + } + + private static String buildJSON(String errMessage) { + // TODO Auto-generated method stub + JSONObject errorJSON = new JSONObject(); + errorJSON.put("error", errMessage); + return errorJSON.toString(); + } + +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/utils/IIIFUtils.java b/rest/src/main/java/cz/incad/kramerius/rest/utils/IIIFUtils.java new file mode 100644 index 0000000000..527bafab7e --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/utils/IIIFUtils.java @@ -0,0 +1,105 @@ +package cz.incad.kramerius.rest.utils; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.SocketException; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response.ResponseBuilder; +import javax.ws.rs.core.StreamingOutput; +import javax.xml.xpath.XPathExpressionException; + +import org.apache.commons.io.IOUtils; +import org.w3c.dom.Document; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.ClientConfig; + +import cz.incad.kramerius.FedoraAccess; +import cz.incad.kramerius.utils.RelsExtHelper; + +public class IIIFUtils { + + public static final Logger LOGGER = Logger.getLogger(IIIFUtils.class.getName()); + + public static String iiifImageEndpoint(String pid, FedoraAccess fedoraAccess) throws IOException { + try { + String url = RelsExtHelper.getRelsExtTilesUrl(pid, fedoraAccess); + if (url == null) + return null; + if (url.trim().equals(RelsExtHelper.CACHE_RELS_EXT_LITERAL)) + return null; + return url.replaceAll("[z|Z]oomify|deepZoom","iiif"); + } catch (XPathExpressionException e) { + throw new IOException(e.getMessage()); + } + } + + public static String iiifImageEndpoint( Document relsExt) throws IOException { + try { + String url = RelsExtHelper.getRelsExtTilesUrl(relsExt); + if (url == null) + return null; + if (url.trim().equals(RelsExtHelper.CACHE_RELS_EXT_LITERAL)) + return null; + return url.replaceAll("[z|Z]oomify|deepZoom","iiif"); + } catch (XPathExpressionException e) { + throw new IOException(e.getMessage()); + } + } + + public static void copyFromImageServer(Client c, String urlString, ByteArrayOutputStream bos, ResponseBuilder builder) throws IOException { + IIIFUtils.copyFromImageServer(c,urlString, bos, builder, null); + } + + public static void copyFromImageServer(Client c, String urlString, ByteArrayOutputStream bos, ResponseBuilder builder,String mimetype) + throws IOException { + c.getProperties().put(ClientConfig.PROPERTY_FOLLOW_REDIRECTS, true); + WebResource r = c.resource(urlString); + ClientResponse clientResponse = r.accept(MediaType.MEDIA_TYPE_WILDCARD).get(ClientResponse.class); + final InputStream input = clientResponse.getEntityInputStream(); + StreamingOutput stream = new StreamingOutput() { + public void write(OutputStream output) throws IOException, WebApplicationException { + try { + IOUtils.copy(input, output); + } catch (IOException e) { + if (e.getCause() != null && e.getCause() instanceof SocketException + && (e.getCause().getMessage().equals("Connection reset") + || e.getCause().getMessage().equals("Broken pipe"))) { + LOGGER.warning("Connection reset probably by client (or by repository)"); + } else { + LOGGER.log(Level.SEVERE, null, e); + } + throw new WebApplicationException(e); + } finally { + LOGGER.fine("closing connection to repository"); + IOUtils.closeQuietly(input); + } + } + }; + builder.entity(stream); + + if (mimetype != null) { + builder.type(mimetype); + } + + // added by filter + //builder.header("Access-Control-Allow-Origin", "*"); + MultivaluedMap headers = clientResponse.getHeaders(); + if (headers.containsKey("Cache-Control")) { + builder.header("Cache-Control", headers.getFirst("Cache-Control")); + } + if (headers.containsKey("Last-Modified")) { + builder.header("Last-Modified", headers.getFirst("Last-Modified")); + } + } + +} diff --git a/search/src/java/cz/incad/Kramerius/StartupServlet.java b/search/src/java/cz/incad/Kramerius/StartupServlet.java index 908c329caa..443c98a4ec 100644 --- a/search/src/java/cz/incad/Kramerius/StartupServlet.java +++ b/search/src/java/cz/incad/Kramerius/StartupServlet.java @@ -44,6 +44,7 @@ import cz.incad.kramerius.pdf.GeneratePDFService; import cz.incad.kramerius.processes.database.MostDesirableDbInitializer; import cz.incad.kramerius.processes.database.ProcessDbInitializer; +import cz.incad.kramerius.rest.oai.db.OAIDBInitializer; import cz.incad.kramerius.security.database.SecurityDbInitializer; import cz.incad.kramerius.service.LifeCycleHookRegistry; import cz.incad.kramerius.service.TextsService; @@ -115,6 +116,9 @@ public void init() throws ServletException { // delete session keys LoggedUserDbHelper.deleteAllSessionKeys(connection); + // Default OAI sets initializer - configuration part + OAIDBInitializer.initDatabase(connection, versionService); + // stores new db version to database if necessary versionService.updateVersionIfOutdated(); diff --git a/shared/common/src/main/java/cz/incad/kramerius/database/impl/res/current.db.version b/shared/common/src/main/java/cz/incad/kramerius/database/impl/res/current.db.version index 8fc3e14c45..1b19111839 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/database/impl/res/current.db.version +++ b/shared/common/src/main/java/cz/incad/kramerius/database/impl/res/current.db.version @@ -1 +1 @@ -6.8.3 \ No newline at end of file +7.0.4 \ No newline at end of file From 5df6adf31e0d681443dd3b0e72f5022182419341 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 12 Feb 2024 13:15:07 +0100 Subject: [PATCH 102/235] Issue https://github.com/ceskaexpedice/ceska-digitalni-knihovna/issues/48 --- .../database/SecurityDbInitializer.java | 59 +- .../security/database/res/initsecdb.sql | 1 + .../database/res/initsecdb_newactions.sql | 83 ++ .../database/res/rename_oldactions.sql | 36 + .../database/StatisticDbInitializer.java | 888 ++++++++---------- 5 files changed, 590 insertions(+), 477 deletions(-) create mode 100644 shared/common/src/main/java/cz/incad/kramerius/security/database/res/initsecdb_newactions.sql create mode 100644 shared/common/src/main/java/cz/incad/kramerius/security/database/res/rename_oldactions.sql diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/database/SecurityDbInitializer.java b/shared/common/src/main/java/cz/incad/kramerius/security/database/SecurityDbInitializer.java index 55675c59fc..28f270c130 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/database/SecurityDbInitializer.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/database/SecurityDbInitializer.java @@ -94,6 +94,8 @@ public static void initDatabase(Connection connection, VersionService versionSer // labels table makeSureThatLabelsTable(connection); + makeSureThatRoleColumnExists(connection); + createNewActions(connection); } else { String v = versionService.getVersion(); @@ -139,6 +141,9 @@ public static void initDatabase(Connection connection, VersionService versionSer // labels table makeSureThatLabelsTable(connection); + makeSureThatRoleColumnExists(connection); + + createNewActions(connection); } else if (versionCondition(v, "=", "5.3.0")) { // right for criteria params manage @@ -165,6 +170,9 @@ public static void initDatabase(Connection connection, VersionService versionSer // labels table makeSureThatLabelsTable(connection); + makeSureThatRoleColumnExists(connection); + createNewActions(connection); + } else if (versionCondition(v, "=", "5.4.0")) { // k4 replication rights @@ -188,6 +196,9 @@ public static void initDatabase(Connection connection, VersionService versionSer // labels table makeSureThatLabelsTable(connection); + makeSureThatRoleColumnExists(connection); + createNewActions(connection); + } else if (versionCondition(v, "=", "5.5.0")) { // mets ndk import @@ -206,6 +217,8 @@ public static void initDatabase(Connection connection, VersionService versionSer // labels table makeSureThatLabelsTable(connection); + makeSureThatRoleColumnExists(connection); + createNewActions(connection); } else if (versionCondition(v, "=", "5.6.0")) { // replikator k3 @@ -222,7 +235,8 @@ public static void initDatabase(Connection connection, VersionService versionSer // labels table makeSureThatLabelsTable(connection); - + makeSureThatRoleColumnExists(connection); + createNewActions(connection); } else if (versionCondition(v, "=", "5.7.0")) { // insert aggregate process right @@ -237,6 +251,8 @@ public static void initDatabase(Connection connection, VersionService versionSer // labels table makeSureThatLabelsTable(connection); + makeSureThatRoleColumnExists(connection); + createNewActions(connection); } else if (versionCondition(v, "=", "5.8.0")) { // insert aggregate process right @@ -248,6 +264,7 @@ public static void initDatabase(Connection connection, VersionService versionSer insertPrintRight(connection); updateUserEntityTable(connection); updateShowItems(connection); + makeSureThatRoleColumnExists(connection); // labels table makeSureThatLabelsTable(connection); @@ -263,6 +280,9 @@ public static void initDatabase(Connection connection, VersionService versionSer // labels table makeSureThatLabelsTable(connection); + makeSureThatRoleColumnExists(connection); + createNewActions(connection); + } else if ((versionCondition(v, ">", "5.9.0")) && (versionCondition(v, "<", "6.3.0"))) { //sort right @@ -273,6 +293,9 @@ public static void initDatabase(Connection connection, VersionService versionSer // labels table makeSureThatLabelsTable(connection); + makeSureThatRoleColumnExists(connection); + + createNewActions(connection); } else if (versionCondition(v, "=", "6.3.0")) { insertPrintRight(connection); @@ -281,6 +304,10 @@ public static void initDatabase(Connection connection, VersionService versionSer // labels table makeSureThatLabelsTable(connection); + makeSureThatRoleColumnExists(connection); + + createNewActions(connection); + } else if (versionCondition(v, "=", "6.6.0")) { updateUserEntityTable(connection); @@ -288,18 +315,27 @@ public static void initDatabase(Connection connection, VersionService versionSer // labels table makeSureThatLabelsTable(connection); + makeSureThatRoleColumnExists(connection); + + createNewActions(connection); + } else if (versionCondition(v, ">", "6.6.0") && versionCondition(v, "<=", "6.6.2")) { updateShowItems(connection); // labels table makeSureThatLabelsTable(connection); + makeSureThatRoleColumnExists(connection); + createNewActions(connection); + } else if (versionCondition(v, ">", "6.6.2")) { // labels table makeSureThatLabelsTable(connection); - } + makeSureThatRoleColumnExists(connection); + createNewActions(connection); + } } @@ -311,6 +347,8 @@ public static void initDatabase(Connection connection, VersionService versionSer // labels table makeSureThatLabelsTable(connection); + updateExistingActions(connection); + } catch (SQLException e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); } catch (IOException e) { @@ -537,6 +575,23 @@ public static void alterTableAddRoleColumn(Connection con) throws SQLException { LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows"); } + private static void updateExistingActions(Connection connection) throws SQLException, IOException { + InputStream is = InitSecurityDatabaseMethodInterceptor.class.getResourceAsStream("res/rename_oldactions.sql"); + JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); + template.setUseReturningKeys(false); + template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); + } + + /** Premena na nove akce */ + // vytvareni + private static void createNewActions(Connection connection) throws SQLException, IOException { + InputStream is = InitSecurityDatabaseMethodInterceptor.class.getResourceAsStream("res/initsecdb_newactions.sql"); + JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); + template.setUseReturningKeys(false); + template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); + } + + public static void updateRolesFromRelations(Connection con) throws SQLException { String update = "UPDATE right_entity " + "SET \"role\"=subquery.gname " + diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/database/res/initsecdb.sql b/shared/common/src/main/java/cz/incad/kramerius/security/database/res/initsecdb.sql index ef1303fe2f..82c494f4ce 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/database/res/initsecdb.sql +++ b/shared/common/src/main/java/cz/incad/kramerius/security/database/res/initsecdb.sql @@ -77,6 +77,7 @@ CREATE TABLE RIGHT_ENTITY ( RIGHTS_CRIT INT, USER_ID INT, GROUP_ID INT, + ROLE TEXT, FIXED_PRIORITY INT, PRIMARY KEY (RIGHT_ID)); CREATE INDEX UUID_IDX ON RIGHT_ENTITY (UUID); diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/database/res/initsecdb_newactions.sql b/shared/common/src/main/java/cz/incad/kramerius/security/database/res/initsecdb_newactions.sql new file mode 100644 index 0000000000..4c8666c309 --- /dev/null +++ b/shared/common/src/main/java/cz/incad/kramerius/security/database/res/initsecdb_newactions.sql @@ -0,0 +1,83 @@ +-- roles +-- kramerius-admin +INSERT INTO group_entity (group_id, gname) SELECT nextval('group_id_sequence'), 'kramerius_admin' WHERE NOT EXISTS ( SELECT group_id FROM group_entity WHERE gname = 'kramerius_admin'); + + +-- akce pro kramerius_admin +-- a_read +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE '), 'uuid:1','a_read', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_pdf_read +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE '), 'uuid:1','a_pdf_read', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_import +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE '), 'uuid:1','a_import', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_delete +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE '), 'uuid:1','a_delete', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_process_edit +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE '), 'uuid:1','a_process_edit', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_process_read +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE '), 'uuid:1','a_process_read', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_owner_process_edit +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE '), 'uuid:1','a_owner_process_edit', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_index +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE '), 'uuid:1','a_index', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_rebuild_processing_index +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE '), 'uuid:1','a_rebuild_processing_index', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_import +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE'), 'uuid:1','a_import', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_accessibility +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE '), 'uuid:1','a_set_accessibility', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_statistics +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE'), 'uuid:1','a_statistics', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_statistics_edit +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE'), 'uuid:1','a_statistics_edit', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_export_replications +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE'), 'uuid:1','a_export_replications', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_import_replications +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE'), 'uuid:1','a_import_replications', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_rights_edit +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE'), 'uuid:1','a_rights_edit', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_criteria_read +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE'), 'uuid:1','a_criteria_read', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- collections_read +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE'), 'uuid:1','a_collections_read', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- collections_edit +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE'), 'uuid:1','a_collections_edit', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_able_tobe_part_of_collections +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE'), 'uuid:1','a_able_tobe_part_of_collections', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_admin_read +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE'), 'uuid:1','a_admin_read', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- generate nkplogs +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE'), 'uuid:1','a_generate_nkplogs', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_roles_edit +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE'), 'uuid:1','a_roles_edit', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- a_users_edit +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE'), 'uuid:1','a_users_edit', 'kramerius_admin',group_id from group_entity WHERE gname = 'kramerius_admin'; + +-- statistics +-- common_users - read statistics +insert into RIGHT_ENTITY(RIGHT_ID, UUID,ACTION, "role",GROUP_ID) SELECT nextval('RIGHT_ID_SEQUENCE'), 'uuid:1','a_statistics', 'common_users',group_id from group_entity WHERE gname = 'common_users'; + + diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/database/res/rename_oldactions.sql b/shared/common/src/main/java/cz/incad/kramerius/security/database/res/rename_oldactions.sql new file mode 100644 index 0000000000..d8c04a67d1 --- /dev/null +++ b/shared/common/src/main/java/cz/incad/kramerius/security/database/res/rename_oldactions.sql @@ -0,0 +1,36 @@ + +-- rename +update right_entity set "action"='a_read' where "action"='read'; + +update right_entity set "action"='a_pdf_read' where "action"='pdf_resource'; + +update right_entity set "action"='a_index' where "action"='reindex'; + +update right_entity set "action"='a_delete' where "action"='delete'; + +update right_entity set "action"='a_process_edit' where "action"='manage_lr_process'; + +update right_entity set "action"='a_import' where "action"='import'; + +update right_entity set "action"='a_set_accessibility' where "action"='setprivate'; + +update right_entity set "action"='a_set_accessibility' where "action"='setpublic'; + +update right_entity set "action"='a_statistics' where "action"='show_statictics'; + +update right_entity set "action"='a_statistics_edit' where "action"='manage_statistics'; + +update right_entity set role='kramerius_admin' where role='k4_admins'; + + +-- delete unused +delete from right_entity where "action"='show_print_menu'; +delete from right_entity where "action"='show_client_print_menu'; +delete from right_entity where "action"='show_client_pdf_menu'; +delete from right_entity where "action"='display_admin_menu'; +delete from right_entity where "action"='criteria_rights_manage'; +delete from right_entity where "action"='convert'; +delete from right_entity where "action"='replicationrights'; +delete from right_entity where "action"='enumerator'; +delete from right_entity where "action"='replikator_periodicals'; +delete from right_entity where "action"='replikator_monographs'; diff --git a/shared/common/src/main/java/cz/incad/kramerius/statistics/database/StatisticDbInitializer.java b/shared/common/src/main/java/cz/incad/kramerius/statistics/database/StatisticDbInitializer.java index 39506c3f88..17a091161b 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/statistics/database/StatisticDbInitializer.java +++ b/shared/common/src/main/java/cz/incad/kramerius/statistics/database/StatisticDbInitializer.java @@ -1,22 +1,3 @@ -/* - * Copyright (C) 2012 Pavel Stastny - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -/** - * - */ package cz.incad.kramerius.statistics.database; import static cz.incad.kramerius.database.cond.ConditionsInterpretHelper.versionCondition; @@ -55,163 +36,37 @@ public class StatisticDbInitializer { public static void initDatabase(Connection connection, VersionService versionService) { try { String version = versionService.getVersion(); + + if (version == null) { - createStatisticTables(connection); - alterStatisticsTableStatAction(connection); - createDatesDurationViews(connection); - alterStatisticsTableSessionId(connection); - // Issue 619 - alterStatisticsAuthorTablePrimaryKey(connection); createFirstFunction(connection); createLastFunction(connection); - createTmpAuthorView(connection); - createAuthorsView(connection); - createLangsView(connection); - - checkAndAddDNNTFlag(connection); - checkAndAddProvidedByDNNTFlag(connection); - checkAndAddEvaluateMap(connection); - checkAndAddUserAttributesMap(connection); - - checkPublisherTables(connection); - checkAndAddSolrDate(connection); - } else if (versionCondition(version, "<", "6.0.0")) { - createStatisticTables(connection); - alterStatisticsTableStatAction(connection); - createDatesDurationViews(connection); - alterStatisticsTableSessionId(connection); - - // Issue 619 - alterStatisticsAuthorTablePrimaryKey(connection); createFirstFunction(connection); createLastFunction(connection); - createTmpAuthorView(connection); - createAuthorsView(connection); - createLangsView(connection); - - checkAndAddDNNTFlag(connection); - checkAndAddProvidedByDNNTFlag(connection); - checkAndAddEvaluateMap(connection); - checkAndAddUserAttributesMap(connection); - - checkPublisherTables(connection); - checkAndAddSolrDate(connection); - } else if (versionCondition(version, "=", "6.0.0")) { - alterStatisticsTableStatAction(connection); - createDatesDurationViews(connection); - alterStatisticsTableSessionId(connection); - - // Issue 619 - alterStatisticsAuthorTablePrimaryKey(connection); createFirstFunction(connection); createLastFunction(connection); - createTmpAuthorView(connection); - createAuthorsView(connection); - createLangsView(connection); - - checkAndAddDNNTFlag(connection); - checkAndAddProvidedByDNNTFlag(connection); - checkAndAddEvaluateMap(connection); - checkAndAddUserAttributesMap(connection); - - checkPublisherTables(connection); - checkAndAddSolrDate(connection); - } else if (versionCondition(version, "=", "6.1.0")) { - alterStatisticsTableSessionId(connection); - - // Issue 619 - alterStatisticsAuthorTablePrimaryKey(connection); createFirstFunction(connection); createLastFunction(connection); - createTmpAuthorView(connection); - createAuthorsView(connection); - createLangsView(connection); - - checkAndAddDNNTFlag(connection); - checkAndAddProvidedByDNNTFlag(connection); - checkAndAddEvaluateMap(connection); - checkAndAddUserAttributesMap(connection); - - checkPublisherTables(connection); - checkAndAddSolrDate(connection); - } else if ((versionCondition(version, ">", "6.1.0")) && (versionCondition(version, "<", "6.5.0"))) { - // Issue 619 - alterStatisticsAuthorTablePrimaryKey(connection); createFirstFunction(connection); createLastFunction(connection); - createTmpAuthorView(connection); - createAuthorsView(connection); - createLangsView(connection); - - checkAndAddDNNTFlag(connection); - checkAndAddProvidedByDNNTFlag(connection); - checkAndAddEvaluateMap(connection); - checkAndAddUserAttributesMap(connection); - - checkPublisherTables(connection); - checkAndAddSolrDate(connection); - } else if (versionCondition(version, ">=", "6.5.0") && (versionCondition(version, "<", "6.6.4"))) { - createFirstFunction(connection); + + createFirstFunction(connection); createLastFunction(connection); - createTmpAuthorView(connection); - createAuthorsView(connection); - createLangsView(connection); - - checkAndAddDNNTFlag(connection); - checkAndAddProvidedByDNNTFlag(connection); - checkAndAddEvaluateMap(connection); - checkAndAddUserAttributesMap(connection); - - checkPublisherTables(connection); - checkAndAddSolrDate(connection); - - - } else if ((versionCondition(version, ">=", "6.6.4")) && (versionCondition(version, "<", "6.6.6"))) { - createTmpAuthorView(connection); - createAuthorsView(connection); - createLangsView(connection); - - checkAndAddDNNTFlag(connection); - checkAndAddProvidedByDNNTFlag(connection); - checkAndAddEvaluateMap(connection); - checkAndAddUserAttributesMap(connection); - - checkPublisherTables(connection); - checkAndAddSolrDate(connection); - - - } else if (versionCondition(version, ">=", "6.6.6")) { - checkAndAddDNNTFlag(connection); - checkAndAddProvidedByDNNTFlag(connection); - checkAndAddEvaluateMap(connection); - checkAndAddUserAttributesMap(connection); - - checkPublisherTables(connection); - checkAndAddSolrDate(connection); } - // check if date column contains index, if not, creates it - checkDateIndex(connection); // check if labels_entity table exists, if not creates it - checkLabelExists(connection); - // check if statistics table contains columns for issn, isbn, ccnb, if not, create them - checkIsbnIssnCcnb(connection); - // check if statistics table contains column for dbversion, if not crates it - checkLogVersionColumn(connection); - - // check if labels exists - checkLabelsColumns(connection); - + //checkLabelExists(connection); + } catch (SQLException e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); } catch (IOException e) { @@ -219,46 +74,118 @@ public static void initDatabase(Connection connection, VersionService versionSer } } - private static void checkLabelsColumns(Connection connection) { - try { - if (!DatabaseUtils.columnExists(connection, "statistics_access_log", "dnnt_labels")) { - InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/initlabelscolumn.sql"); - JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); - template.setUseReturningKeys(false); - template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); - } - } catch (SQLException | IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } - } - - private static void checkLogVersionColumn(Connection connection) { - try { - if (!DatabaseUtils.columnExists(connection, "statistics_access_log", "dbversion")) { - InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/initdebversioncolumn.sql"); - JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); - template.setUseReturningKeys(false); - template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); - } - } catch (SQLException | IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } - } - - private static void checkIsbnIssnCcnb(Connection connection) { - try { - if (!DatabaseUtils.columnExists(connection, "statistic_access_log_detail", "issn")) { - InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/initidentifierscolumn.sql"); - JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); - template.setUseReturningKeys(false); - template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); - } - } catch (SQLException | IOException e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - } - } +// private static void changeStatisticsForeignKeys(Connection connection) throws SQLException, IOException { +// InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/rebuildconstraints.sql"); +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); +// template.setUseReturningKeys(false); +// template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); +// } + + + +// private static void checkStatisticsView_Lang(Connection connection) { +// try { +// if (!DatabaseUtils.materializedViewExists(connection, "_lang") || !DatabaseUtils.materializedViewExists(connection, "_statistic_access_log_detail_lang_not_null")) { +// LOGGER.info("Preparing materialized views (lang)"); +// InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/initlang.sql"); +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); +// template.setUseReturningKeys(false); +// template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); +// } else { +// LOGGER.info("Refreshing materialized views (lang)"); +// InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/refreshlang.sql"); +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); +// template.setUseReturningKeys(false); +// template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); +// } +// } catch (SQLException | IOException e) { +// LOGGER.log(Level.SEVERE, e.getMessage(), e); +// } +// } +// +// +// +// private static void checkStatisticsView_Authors(Connection connection) { +// try { +// if (!DatabaseUtils.materializedViewExists(connection, "_authors")) { +// LOGGER.info("Preparing materialized views (authors)"); +// InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/initauthors.sql"); +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); +// template.setUseReturningKeys(false); +// template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); +// } else { +// LOGGER.info("Refreshing materialized views (authors)"); +// InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/refreshauthors.sql"); +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); +// template.setUseReturningKeys(false); +// template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); +// } +// } catch (SQLException | IOException e) { +// LOGGER.log(Level.SEVERE, e.getMessage(), e); +// } +// } +// +// private static void checkStatisticsView_Models(Connection connection) { +// try { +// if (!DatabaseUtils.materializedViewExists(connection, "_model_monograph")) { +// LOGGER.info("Preparing materialized views (models)"); +// InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/initmodels.sql"); +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); +// template.setUseReturningKeys(false); +// template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); +// } else { +// LOGGER.info("Refreshing materialized views (models)"); +// InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/refreshmodels.sql"); +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); +// template.setUseReturningKeys(false); +// template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); +// } +// } catch (SQLException | IOException e) { +// LOGGER.log(Level.SEVERE, e.getMessage(), e); +// } +// } + +// private static void checkLabelsColumns(Connection connection) { +// try { +// if (!DatabaseUtils.columnExists(connection, "statistics_access_log", "dnnt_labels")) { +// InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/initlabelscolumn.sql"); +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); +// template.setUseReturningKeys(false); +// template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); +// } +// } catch (SQLException | IOException e) { +// LOGGER.log(Level.SEVERE, e.getMessage(), e); +// } +// } +// +// private static void checkLogVersionColumn(Connection connection) { +// try { +// if (!DatabaseUtils.columnExists(connection, "statistics_access_log", "dbversion")) { +// InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/initdebversioncolumn.sql"); +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); +// template.setUseReturningKeys(false); +// template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); +// } +// } catch (SQLException | IOException e) { +// LOGGER.log(Level.SEVERE, e.getMessage(), e); +// } +// } +// +// private static void checkIsbnIssnCcnb(Connection connection) { +// try { +// if (!DatabaseUtils.columnExists(connection, "statistic_access_log_detail", "issn")) { +// InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/initidentifierscolumn.sql"); +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); +// template.setUseReturningKeys(false); +// template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); +// } +// } catch (SQLException | IOException e) { +// LOGGER.log(Level.SEVERE, e.getMessage(), e); +// } +// } private static void checkLabelExists(Connection connection) { + EMBEDDED_LABELS.stream().forEach(label -> { List labels = new JDBCQueryTemplate(connection, false) { @Override @@ -285,191 +212,199 @@ public boolean handleRow(ResultSet rs, List returnsList) throws SQLExcep ); } - /** - * @param con - * @throws SQLException - */ - private static void createDatesDurationViews(Connection con) throws SQLException { - List commands = new ArrayList(); - commands.add(new JDBCCommand() { - - @Override - public Object executeJDBCCommand(Connection con) throws SQLException { - PreparedStatement prepareStatement = con.prepareStatement( - "create view flat_statistic_access_log_detail_map as select record_id, min(detail_id) as detail_id from statistic_access_log_detail group by record_id"); - int r = prepareStatement.executeUpdate(); - LOGGER.log(Level.FINEST, "CREATE VIEW: updated rows {0}", r); - return null; - } - }); - commands.add(new JDBCCommand() { - - @Override - public Object executeJDBCCommand(Connection con) throws SQLException { - PreparedStatement prepareStatement = con.prepareStatement( - "create view flat_statistic_access_log_detail as select fa.detail_id, fa.record_id, sa.pid, sa.model, sa.issued_date, sa.rights, sa.lang,sa.title from flat_statistic_access_log_detail_map fa join statistic_access_log_detail sa using(detail_id)"); - int r = prepareStatement.executeUpdate(); - LOGGER.log(Level.FINEST, "CREATE VIEW: updated rows {0}", r); - return null; - } - }); - - new JDBCTransactionTemplate(con, false).updateWithTransaction(commands); - } - - private static void alterStatisticsTableStatAction(Connection con) throws SQLException { - PreparedStatement prepareStatement = con - .prepareStatement("ALTER TABLE statistics_access_log ADD COLUMN STAT_ACTION VARCHAR(255);"); - try { - int r = prepareStatement.executeUpdate(); - LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); - } finally { - DatabaseUtils.tryClose(prepareStatement); - } - } - - private static void checkAndAddDNNTFlag(Connection con) throws SQLException { - if (!DatabaseUtils.columnExists(con, "statistics_access_log", "dnnt")) { - PreparedStatement prepareStatement = con.prepareStatement("ALTER TABLE statistics_access_log ADD COLUMN dnnt BOOLEAN ;"); - try { - int r = prepareStatement.executeUpdate(); - LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); - } finally { - DatabaseUtils.tryClose(prepareStatement); - } - } - } - - private static void checkAndAddProvidedByDNNTFlag(Connection con) throws SQLException { - if (!DatabaseUtils.columnExists(con, "statistics_access_log", "providedByDNNT")) { - PreparedStatement prepareStatement = con.prepareStatement("ALTER TABLE statistics_access_log ADD COLUMN providedByDNNT BOOLEAN ;"); - try { - int r = prepareStatement.executeUpdate(); - LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); - } finally { - DatabaseUtils.tryClose(prepareStatement); - } - } - } - - private static void checkAndAddEvaluateMap(Connection con) throws SQLException { - if (!DatabaseUtils.columnExists(con, "statistics_access_log", "evaluateMap")) { - PreparedStatement prepareStatement = con.prepareStatement("ALTER TABLE statistics_access_log ADD COLUMN evaluateMap text;"); - try { - int r = prepareStatement.executeUpdate(); - LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); - } finally { - DatabaseUtils.tryClose(prepareStatement); - } - } - } - - private static void checkAndAddUserAttributesMap(Connection con) throws SQLException { - if (!DatabaseUtils.columnExists(con, "statistics_access_log", "userSessionAttributes")) { - PreparedStatement prepareStatement = con.prepareStatement("ALTER TABLE statistics_access_log ADD COLUMN userSessionAttributes text;"); - try { - int r = prepareStatement.executeUpdate(); - LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); - } finally { - DatabaseUtils.tryClose(prepareStatement); - } - } - } - - private static void checkAndAddSolrDate(Connection con) throws SQLException { - if (!DatabaseUtils.columnExists(con, "statistic_access_log_detail", "solr_date")) { - PreparedStatement prepareStatement = con.prepareStatement("ALTER TABLE statistic_access_log_detail ADD COLUMN solr_date TEXT;"); - try { - int r = prepareStatement.executeUpdate(); - LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); - } finally { - DatabaseUtils.tryClose(prepareStatement); - } - } - } - - private static void checkDateIndex(Connection con) throws SQLException { - if (!DatabaseUtils.indexExists(con, "statistics_access_log", "date")) { - PreparedStatement prepareStatement = con.prepareStatement("CREATE INDEX date_index on statistics_access_log(date);"); - try { - int r = prepareStatement.executeUpdate(); - LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); - } finally { - DatabaseUtils.tryClose(prepareStatement); - } - } - } - - private static void alterStatisticsTableSessionId(Connection con) throws SQLException { - PreparedStatement prepareStatement = con - .prepareStatement("ALTER TABLE statistics_access_log ADD COLUMN SESSION_ID VARCHAR(255);"); - try { - int r = prepareStatement.executeUpdate(); - LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); - } finally { - DatabaseUtils.tryClose(prepareStatement); - } - } +// /** +// * @param con +// * @throws SQLException +// */ +// private static void createDatesDurationViews(Connection con) throws SQLException { +// List commands = new ArrayList(); +// commands.add(new JDBCCommand() { +// +// @Override +// public Object executeJDBCCommand(Connection con) throws SQLException { +// PreparedStatement prepareStatement = con.prepareStatement( +// "create view flat_statistic_access_log_detail_map as select record_id, min(detail_id) as detail_id from statistic_access_log_detail group by record_id"); +// int r = prepareStatement.executeUpdate(); +// LOGGER.log(Level.FINEST, "CREATE VIEW: updated rows {0}", r); +// return null; +// } +// }); +// commands.add(new JDBCCommand() { +// +// @Override +// public Object executeJDBCCommand(Connection con) throws SQLException { +// PreparedStatement prepareStatement = con.prepareStatement( +// "create view flat_statistic_access_log_detail as select fa.detail_id, fa.record_id, sa.pid, sa.model, sa.issued_date, sa.rights, sa.lang,sa.title from flat_statistic_access_log_detail_map fa join statistic_access_log_detail sa using(detail_id)"); +// int r = prepareStatement.executeUpdate(); +// LOGGER.log(Level.FINEST, "CREATE VIEW: updated rows {0}", r); +// return null; +// } +// }); +// +// new JDBCTransactionTemplate(con, false).updateWithTransaction(commands); +// } + +// private static void checkStatisticsTableStatAction(Connection con) throws SQLException { +// if (!DatabaseUtils.columnExists(con, "statistics_access_log", "STAT_ACTION")) { +// PreparedStatement prepareStatement = con +// .prepareStatement("ALTER TABLE statistics_access_log ADD COLUMN STAT_ACTION VARCHAR(255);"); +// try { +// int r = prepareStatement.executeUpdate(); +// LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); +// } finally { +// DatabaseUtils.tryClose(prepareStatement); +// } +// } +// } + +// private static void checkAndAddDNNTFlag(Connection con) throws SQLException { +// if (!DatabaseUtils.columnExists(con, "statistics_access_log", "dnnt")) { +// PreparedStatement prepareStatement = con.prepareStatement("ALTER TABLE statistics_access_log ADD COLUMN dnnt BOOLEAN ;"); +// try { +// int r = prepareStatement.executeUpdate(); +// LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); +// } finally { +// DatabaseUtils.tryClose(prepareStatement); +// } +// } +// } + +// private static void checkAndAddProvidedByDNNTFlag(Connection con) throws SQLException { +// if (!DatabaseUtils.columnExists(con, "statistics_access_log", "providedByDNNT")) { +// PreparedStatement prepareStatement = con.prepareStatement("ALTER TABLE statistics_access_log ADD COLUMN providedByDNNT BOOLEAN ;"); +// try { +// int r = prepareStatement.executeUpdate(); +// LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); +// } finally { +// DatabaseUtils.tryClose(prepareStatement); +// } +// } +// } + +// private static void checkAndAddEvaluateMap(Connection con) throws SQLException { +// if (!DatabaseUtils.columnExists(con, "statistics_access_log", "evaluateMap")) { +// PreparedStatement prepareStatement = con.prepareStatement("ALTER TABLE statistics_access_log ADD COLUMN evaluateMap text;"); +// try { +// int r = prepareStatement.executeUpdate(); +// LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); +// } finally { +// DatabaseUtils.tryClose(prepareStatement); +// } +// } +// } + +// private static void checkAndAddUserAttributesMap(Connection con) throws SQLException { +// if (!DatabaseUtils.columnExists(con, "statistics_access_log", "userSessionAttributes")) { +// PreparedStatement prepareStatement = con.prepareStatement("ALTER TABLE statistics_access_log ADD COLUMN userSessionAttributes text;"); +// try { +// int r = prepareStatement.executeUpdate(); +// LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); +// } finally { +// DatabaseUtils.tryClose(prepareStatement); +// } +// } +// } + +// private static void checkAndAddSolrDate(Connection con) throws SQLException { +// if (!DatabaseUtils.columnExists(con, "statistic_access_log_detail", "solr_date")) { +// PreparedStatement prepareStatement = con.prepareStatement("ALTER TABLE statistic_access_log_detail ADD COLUMN solr_date TEXT;"); +// try { +// int r = prepareStatement.executeUpdate(); +// LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); +// } finally { +// DatabaseUtils.tryClose(prepareStatement); +// } +// } +// } + +// private static void checkDateIndex(Connection con) throws SQLException { +// if (!DatabaseUtils.indexExists(con, "statistics_access_log", "date")) { +// PreparedStatement prepareStatement = con.prepareStatement("CREATE INDEX date_index on statistics_access_log(date);"); +// try { +// int r = prepareStatement.executeUpdate(); +// LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); +// } finally { +// DatabaseUtils.tryClose(prepareStatement); +// } +// } +// } + + +// private static void alterStatisticsTableSessionId(Connection con) throws SQLException { +// +// +// PreparedStatement prepareStatement = con +// .prepareStatement("ALTER TABLE statistics_access_log ADD COLUMN SESSION_ID VARCHAR(255);"); +// try { +// int r = prepareStatement.executeUpdate(); +// LOGGER.log(Level.FINEST, "ALTER TABLE: updated rows {0}", r); +// } finally { +// DatabaseUtils.tryClose(prepareStatement); +// } +// } // change pk // Issue 619 - private static void alterStatisticsAuthorTablePrimaryKey(final Connection con) throws SQLException { - - JDBCCommand deletePKCommand = new JDBCCommand() { - - @Override - public Object executeJDBCCommand(Connection con) throws SQLException { - JDBCUpdateTemplate template = new JDBCUpdateTemplate(con, false); - template.setUseReturningKeys(false); - template.executeUpdate( - "ALTER TABLE statistic_access_log_detail_authors DROP CONSTRAINT statistic_access_log_detail_authors_pkey", - new Object[0]); - return null; - } - }; - - JDBCCommand createPKCommand = new JDBCCommand() { - - @Override - public Object executeJDBCCommand(Connection con) throws SQLException { - JDBCUpdateTemplate template = new JDBCUpdateTemplate(con, false); - template.setUseReturningKeys(false); - template.executeUpdate("ALTER TABLE statistic_access_log_detail_authors ADD PRIMARY KEY (author_id)", - new Object[0]); - return null; - } - }; - - new JDBCTransactionTemplate(con, false).updateWithTransaction(deletePKCommand, createPKCommand); - - } - - /** - * @param connection - * @throws IOException - * @throws SQLException - */ - private static void createStatisticTables(Connection connection) throws SQLException, IOException { - InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/initstatisticsdb.sql"); - JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); - template.setUseReturningKeys(false); - template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); - } - - /** - * Check if publisher table exists - * @param connection Connection - * @throws SQLException - * @throws IOException - */ - private static void checkPublisherTables(Connection connection) throws SQLException, IOException { - if (!DatabaseUtils.tableExists(connection, "STATISTIC_ACCESS_LOG_DETAIL_PUBLISHERS")) { - InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/initpublishersdb.sql"); - JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); - template.setUseReturningKeys(false); - template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); - } - } +// private static void alterStatisticsAuthorTablePrimaryKey(final Connection con) throws SQLException { +// +// JDBCCommand deletePKCommand = new JDBCCommand() { +// +// @Override +// public Object executeJDBCCommand(Connection con) throws SQLException { +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(con, false); +// template.setUseReturningKeys(false); +// template.executeUpdate( +// "ALTER TABLE statistic_access_log_detail_authors DROP CONSTRAINT statistic_access_log_detail_authors_pkey", +// new Object[0]); +// return null; +// } +// }; +// +// JDBCCommand createPKCommand = new JDBCCommand() { +// +// @Override +// public Object executeJDBCCommand(Connection con) throws SQLException { +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(con, false); +// template.setUseReturningKeys(false); +// template.executeUpdate("ALTER TABLE statistic_access_log_detail_authors ADD PRIMARY KEY (author_id)", +// new Object[0]); +// return null; +// } +// }; +// +// new JDBCTransactionTemplate(con, false).updateWithTransaction(deletePKCommand, createPKCommand); +// +// } + +// /** +// * @param connection +// * @throws IOException +// * @throws SQLException +// */ +// private static void createStatisticTables(Connection connection) throws SQLException, IOException { +// if (!DatabaseUtils.tableExists(connection, "STATISTICS_ACCESS_LOG")) { +// InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/initstatisticsdb.sql"); +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); +// template.setUseReturningKeys(false); +// template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); +// +// } +// } + +// /** +// * Check if publisher table exists +// * @param connection Connection +// * @throws SQLException +// * @throws IOException +// */ +// private static void checkPublisherTables(Connection connection) throws SQLException, IOException { +// if (!DatabaseUtils.tableExists(connection, "STATISTIC_ACCESS_LOG_DETAIL_PUBLISHERS")) { +// InputStream is = StatisticDbInitializer.class.getResourceAsStream("res/initpublishersdb.sql"); +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(connection, false); +// template.setUseReturningKeys(false); +// template.executeUpdate(IOUtils.readAsString(is, Charset.forName("UTF-8"), true)); +// } +// } /** * Create first aggregation function first(col) @@ -547,108 +482,111 @@ public Object executeJDBCCommand(Connection con) throws SQLException { new JDBCTransactionTemplate(connection, false).updateWithTransaction(lastAgg, last); } - private static void createTmpAuthorView(Connection connection) throws SQLException, IOException { - JDBCCommand tmpAuthorsView = new JDBCCommand() { - - @Override - public Object executeJDBCCommand(Connection con) throws SQLException { - JDBCUpdateTemplate template = new JDBCUpdateTemplate(con, false); - template.setUseReturningKeys(false); - - template.executeUpdate( - "CREATE OR REPLACE VIEW _tmp_authors_view AS " - + "SELECT first(record_id) as record_id, " - + "first(dta.pid) as pid, " - + "first(model) as model, " - + "first(session_id) as session_id " - + "FROM statistic_access_log_detail_authors auth " - + "JOIN statistics_access_log sta USING (record_id) " - + "JOIN statistic_access_log_detail dta USING(record_id) " - + "GROUP BY record_id;" - , new Object[0]); - return null; - } - }; - new JDBCTransactionTemplate(connection, false).updateWithTransaction(tmpAuthorsView); - } - - private static void createAuthorsView(Connection connection) throws SQLException, IOException { - JDBCCommand authorsView = new JDBCCommand() { - - @Override - public Object executeJDBCCommand(Connection con) throws SQLException { - try { - JDBCUpdateTemplate dropTemplate = new JDBCUpdateTemplate(con, false); - dropTemplate.setUseReturningKeys(false); - dropTemplate.executeUpdate("DROP VIEW IF EXISTS _authors_view"); - } catch (SQLException e) { - LOGGER.info("Cannot DROP VIEW _authors_view:" + e); - } - - JDBCUpdateTemplate template = new JDBCUpdateTemplate(con, false); - template.setUseReturningKeys(false); - template.executeUpdate( - "CREATE or REPLACE VIEW _authors_view AS " + - "SELECT record_id, " - + "author_id, " - + "author_name, " - + "dta.pid as pid, " - + "model, " - + "session_id, " - + "date, " - + "rights, " - + "stat_action, " - + "remote_ip_address " - + "FROM statistic_access_log_detail_authors auth " - + "JOIN statistics_access_log sta USING (record_id) " - + "JOIN statistic_access_log_detail dta USING(record_id) " - + "JOIN _tmp_authors_view USING (record_id, model, session_id);" - , new Object[0]); - return null; - } - }; - - new JDBCTransactionTemplate(connection, false).updateWithTransaction(authorsView); - } - - private static void createLangsView(Connection connection) throws SQLException, IOException { - JDBCCommand langsView = new JDBCCommand() { - - @Override - public Object executeJDBCCommand(Connection con) throws SQLException { - try { - JDBCUpdateTemplate dropTemplate = new JDBCUpdateTemplate(con, false); - dropTemplate.setUseReturningKeys(false); - dropTemplate.executeUpdate("DROP VIEW IF EXISTS _langs_view"); - } catch (SQLException e) { - LOGGER.info("Cannot DROP VIEW _langs_view:" + e); - } - - JDBCUpdateTemplate template = new JDBCUpdateTemplate(con, false); - template.setUseReturningKeys(false); - template.executeUpdate( - "CREATE or REPLACE VIEW _langs_view AS " + - "(SELECT pid, model, session_id, date, rights, stat_action, CASE WHEN lang1 IS NULL THEN lang2 ELSE lang1 END as lang, remote_ip_address, t1.record_id " - + "FROM " - + "(SELECT * FROM " - + "(SELECT sta.record_id as record_id, dta.pid as pid, dta.model as model, sta.session_id as session_id, sta.date as date, dta.rights as rights, sta.stat_action as stat_action,dta.lang as lang1, remote_ip_address as remote_ip_address " - + "FROM statistics_access_log sta " - + "JOIN statistic_access_log_detail dta USING (record_id)) AS tmp " - + "WHERE (tmp.model = 'article') OR (tmp.model = 'page' AND ((tmp.record_id, 'periodical') in " - + "(SELECT sta.record_id as record_id, dta.model as model " - + "FROM statistics_access_log sta " - + "JOIN statistic_access_log_detail dta USING (record_id)))) " - + "OR (tmp.model = 'monograph') OR (tmp.model = 'archive') OR (tmp.model = 'manuscript') OR (tmp.model = 'sheetmusic') OR (tmp.model = 'soundrecording') OR (tmp.model = 'graphic') OR (tmp.model = 'map')) as T1 " - + "LEFT JOIN " - + "(SELECT sta.record_id as record_id, dta.lang as lang2 " - + "FROM statistics_access_log sta " - + "JOIN statistic_access_log_detail dta USING (record_id) " - + "WHERE dta.model = 'periodicalitem') as T2 " - + "ON (t1.lang1 IS NULL AND t1.model = 'page' AND t1.record_id = t2.record_id));", new Object[0]); - return null; - } - }; - - new JDBCTransactionTemplate(connection, false).updateWithTransaction(langsView); - } +// private static void createTmpAuthorView(Connection connection) throws SQLException, IOException { +// JDBCCommand tmpAuthorsView = new JDBCCommand() { +// +// @Override +// public Object executeJDBCCommand(Connection con) throws SQLException { +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(con, false); +// template.setUseReturningKeys(false); +// +// template.executeUpdate( +// "CREATE OR REPLACE VIEW _tmp_authors_view AS " +// + "SELECT first(record_id) as record_id, " +// + "first(dta.pid) as pid, " +// + "first(model) as model, " +// + "first(session_id) as session_id " +// + "FROM statistic_access_log_detail_authors auth " +// + "JOIN statistics_access_log sta USING (record_id) " +// + "JOIN statistic_access_log_detail dta USING(record_id) " +// + "GROUP BY record_id;" +// , new Object[0]); +// return null; +// } +// }; +// new JDBCTransactionTemplate(connection, false).updateWithTransaction(tmpAuthorsView); +// } + +// private static void createAuthorsView(Connection connection) throws SQLException, IOException { +// JDBCCommand authorsView = new JDBCCommand() { +// +// @Override +// public Object executeJDBCCommand(Connection con) throws SQLException { +// try { +// JDBCUpdateTemplate dropTemplate = new JDBCUpdateTemplate(con, false); +// dropTemplate.setUseReturningKeys(false); +// dropTemplate.executeUpdate("DROP VIEW IF EXISTS _authors_view"); +// } catch (SQLException e) { +// LOGGER.info("Cannot DROP VIEW _authors_view:" + e); +// } +// +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(con, false); +// template.setUseReturningKeys(false); +// template.executeUpdate( +// "CREATE or REPLACE VIEW _authors_view AS " + +// "SELECT record_id, " +// + "author_id, " +// + "author_name, " +// + "dta.pid as pid, " +// + "model, " +// + "session_id, " +// + "date, " +// + "rights, " +// + "stat_action, " +// + "remote_ip_address " +// + "FROM statistic_access_log_detail_authors auth " +// + "JOIN statistics_access_log sta USING (record_id) " +// + "JOIN statistic_access_log_detail dta USING(record_id) " +// + "JOIN _tmp_authors_view USING (record_id, model, session_id);" +// , new Object[0]); +// return null; +// } +// }; +// +// new JDBCTransactionTemplate(connection, false).updateWithTransaction(authorsView); +// } + + // mno... bude se muset prepsat +// private static void createLangsView(Connection connection) throws SQLException, IOException { +// JDBCCommand langsView = new JDBCCommand() { +// +// @Override +// public Object executeJDBCCommand(Connection con) throws SQLException { +// try { +// JDBCUpdateTemplate dropTemplate = new JDBCUpdateTemplate(con, false); +// dropTemplate.setUseReturningKeys(false); +// dropTemplate.executeUpdate("DROP VIEW IF EXISTS _langs_view"); +// } catch (SQLException e) { +// LOGGER.info("Cannot DROP VIEW _langs_view:" + e); +// } +// +// JDBCUpdateTemplate template = new JDBCUpdateTemplate(con, false); +// template.setUseReturningKeys(false); +// template.executeUpdate( +// "CREATE or REPLACE VIEW _langs_view AS " + +// "(SELECT pid, model, session_id, date, rights, stat_action, CASE WHEN lang1 IS NULL THEN lang2 ELSE lang1 END as lang, remote_ip_address, t1.record_id " +// + "FROM " +// + "(SELECT * FROM " +// + "(SELECT sta.record_id as record_id, dta.pid as pid, dta.model as model, sta.session_id as session_id, sta.date as date, dta.rights as rights, sta.stat_action as stat_action,dta.lang as lang1, remote_ip_address as remote_ip_address " +// + "FROM statistics_access_log sta " +// + "JOIN statistic_access_log_detail dta USING (record_id)) AS tmp " +// + "WHERE (tmp.model = 'article') OR (tmp.model = 'page' AND ((tmp.record_id, 'periodical') in " +// + "(SELECT sta.record_id as record_id, dta.model as model " +// + "FROM statistics_access_log sta " +// + "JOIN statistic_access_log_detail dta USING (record_id)))) " +// + "OR (tmp.model = 'monograph') OR (tmp.model = 'archive') OR (tmp.model = 'manuscript') OR (tmp.model = 'sheetmusic') OR (tmp.model = 'soundrecording') OR (tmp.model = 'graphic') OR (tmp.model = 'map')) as T1 " +// + "LEFT JOIN " +// + "(SELECT sta.record_id as record_id, dta.lang as lang2 " +// + "FROM statistics_access_log sta " +// + "JOIN statistic_access_log_detail dta USING (record_id) " +// + "WHERE dta.model = 'periodicalitem') as T2 " +// + "ON (t1.lang1 IS NULL AND t1.model = 'page' AND t1.record_id = t2.record_id));", new Object[0]); +// return null; +// } +// }; +// +// new JDBCTransactionTemplate(connection, false).updateWithTransaction(langsView); +// } + + } From 36b7811102f04014871dfe95e6f2cc5940aebcd4 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 12 Feb 2024 14:54:49 +0100 Subject: [PATCH 103/235] Config resource --- .../rest/apiNew/admin/v10/ConfigResource.java | 223 +++++++++++++++--- 1 file changed, 186 insertions(+), 37 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ConfigResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ConfigResource.java index a31fbd8a6b..c3718842a1 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ConfigResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ConfigResource.java @@ -1,24 +1,37 @@ package cz.incad.kramerius.rest.apiNew.admin.v10; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.inject.Provider; +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.POST; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.WebApplicationException; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.commons.lang3.StringEscapeUtils; +import org.json.JSONArray; +import org.json.JSONObject; + import com.google.inject.Inject; + +import cz.incad.kramerius.ObjectPidsPath; import cz.incad.kramerius.rest.apiNew.ConfigManager; -import cz.incad.kramerius.rest.apiNew.exceptions.BadRequestException; import cz.incad.kramerius.rest.apiNew.exceptions.ForbiddenException; import cz.incad.kramerius.rest.apiNew.exceptions.InternalErrorException; -import cz.incad.kramerius.security.Role; +import cz.incad.kramerius.security.SecuredActions; +import cz.incad.kramerius.security.SpecialObjects; import cz.incad.kramerius.security.User; -import cz.incad.kramerius.utils.StringUtils; -import org.json.JSONObject; - -import javax.inject.Provider; -import javax.ws.rs.*; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.Response; -import java.util.Arrays; -import java.util.List; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; /** * @see cz.incad.kramerius.rest.api.k5.client.info.InfoResource @@ -35,28 +48,76 @@ public class ConfigResource extends AdminApiResource { Provider userProvider; - //TODO: prejmenovat role podle spravy uctu - private static final String ROLE_WRITE_CONFIG = "kramerius_admin"; - private static final String ROLE_READ_CONFIG = "kramerius_admin"; + boolean permitConfig(User user) { + if (user != null) + return this.rightsResolver.isActionAllowed(user, + SecuredActions.A_ADMIN_READ.getFormalName(), + SpecialObjects.REPOSITORY.getPid(), null, + ObjectPidsPath.REPOSITORY_PATH).flag(); + else + return false; + } @PUT + @Path("/") + @Consumes(MediaType.APPLICATION_JSON) + public Response setProperties(String values) { + try { + User user1 = this.userProvider.get(); + if (permitConfig(user1)) { + JSONObject obj = new JSONObject(values); + Map map = new HashMap<>(); + obj.keySet().forEach(key-> { + map.put(key.toString(), obj.getString(key.toString())); + }); + configService.setProperties(map); + return Response.ok(obj.toString()).build(); + } else { + throw new ForbiddenException("user '%s' is not allowed to manage config ", user1.getLoginname()); //403 + } + } catch (WebApplicationException e) { + throw e; + } catch (Throwable e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new InternalErrorException(e.getMessage()); + } + } + + + @DELETE @Path("/{key}") @Consumes(MediaType.TEXT_PLAIN) - public Response setProperty(@PathParam("key") String key, String value) { + public Response deleteProperty(@PathParam("key") String key) { try { - //authentication User user1 = this.userProvider.get(); - List roles = Arrays.stream(user1.getGroups()).map(Role::getName).collect(Collectors.toList()); - //AuthenticatedUser user = getAuthenticatedUserByOauth(); - String role = ROLE_WRITE_CONFIG; - if (!roles.contains(role)) { - throw new ForbiddenException("user '%s' is not allowed to manage config (missing role '%s')", user1.getLoginname(), role); //403 + if (permitConfig(user1)) { + configService.deleteProperty(key); + return Response.ok().build(); + } else { + throw new ForbiddenException("user '%s' is not allowed to manage config ", user1.getLoginname()); //403 } - if (!StringUtils.isAnyString(value)) { - throw new BadRequestException("empty value for key '%s'", key); + } catch (WebApplicationException e) { + throw e; + } catch (Throwable e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new InternalErrorException(e.getMessage()); + } + } + + + + @PUT + @Path("/{key}") + @Consumes(MediaType.TEXT_PLAIN) + public Response setProperty(@PathParam("key") String key, String value) { + try { + User user1 = this.userProvider.get(); + if (permitConfig(user1)) { + configService.setProperty(key, value); + return Response.ok().build(); + } else { + throw new ForbiddenException("user '%s' is not allowed to manage config ", user1.getLoginname()); //403 } - configService.setProperty(key, value); - return Response.ok().build(); } catch (WebApplicationException e) { throw e; } catch (Throwable e) { @@ -65,22 +126,59 @@ public Response setProperty(@PathParam("key") String key, String value) { } } + public static String escapeCharacters(String input) { + StringBuilder result = new StringBuilder(); + + for (char c : input.toCharArray()) { + if (c == '.' || c == '^' || c == '$' || c == '*' || c == '+' || c == '?' || c == '[' || c == ']' || c == '(' || c == ')') { + result.append('\\').append(c); + } else { + result.append(c); + } + } + + return result.toString(); + } + + + @GET + @Path("/keys/{prefix}") + @Produces(MediaType.APPLICATION_JSON) + public Response getKeys(@PathParam("prefix") String prefix) { + try { + User user1 = this.userProvider.get(); + if (permitConfig(user1)) { + List keys = configService.getKeysByRegularExpression("^"+escapeCharacters(prefix)); + JSONArray jsonArray = new JSONArray(); + keys.stream().forEach(jsonArray::put); + return Response.ok(jsonArray.toString()).build(); + } else { + throw new ForbiddenException("user '%s' is not allowed to manage config ", user1.getLoginname()); //403 + } + } catch (WebApplicationException e) { + throw e; + } catch (Throwable e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new InternalErrorException(e.getMessage()); + } + } + + + @GET @Path("/{key}") @Produces(MediaType.APPLICATION_JSON) public Response getProperty(@PathParam("key") String key) { try { - //authentication User user1 = this.userProvider.get(); - List roles = Arrays.stream(user1.getGroups()).map(Role::getName).collect(Collectors.toList()); - String role = ROLE_WRITE_CONFIG; - if (!roles.contains(role)) { - throw new ForbiddenException("user '%s' is not allowed to manage config (missing role '%s')", user1.getLoginname(), role); //403 + if (permitConfig(user1)) { + String value = configService.getProperty(key); + JSONObject json = new JSONObject(); + json.put(key, value); + return Response.ok(json).build(); + } else { + throw new ForbiddenException("user '%s' is not allowed to manage config ", user1.getLoginname()); //403 } - String value = configService.getProperty(key); - JSONObject json = new JSONObject(); - json.put(key, value); - return Response.ok(json).build(); } catch (WebApplicationException e) { throw e; } catch (Throwable e) { @@ -89,4 +187,55 @@ public Response getProperty(@PathParam("key") String key) { } } + + + @POST + @Path("/getProperties") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public Response getProperties(String values) { + try { + User user1 = this.userProvider.get(); + if (permitConfig(user1)) { + JSONObject obj = new JSONObject(values); + Map map = configService.getProperties(obj.keySet()); + JSONObject retval = new JSONObject(map); + return Response.ok(retval).build(); + } else { + throw new ForbiddenException("user '%s' is not allowed to manage config ", user1.getLoginname()); //403 + } + } catch (WebApplicationException e) { + throw e; + } catch (Throwable e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new InternalErrorException(e.getMessage()); + } + + } + + + @POST + @Path("/deleteProperties") + @Consumes(MediaType.APPLICATION_JSON) + @Produces(MediaType.APPLICATION_JSON) + public Response deleteProperties(String values) { + try { + User user1 = this.userProvider.get(); + if (permitConfig(user1)) { + JSONObject obj = new JSONObject(values); + configService.deleteProperties(obj.keySet()); + return Response.ok(obj.toString()).build(); + } else { + throw new ForbiddenException("user '%s' is not allowed to manage config ", user1.getLoginname()); //403 + } + } catch (WebApplicationException e) { + throw e; + } catch (Throwable e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new InternalErrorException(e.getMessage()); + } + + } + + } From ba7de65360e8ffd42430c9bbb8b3f1ef7af79832 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 12 Feb 2024 19:57:07 +0100 Subject: [PATCH 104/235] Issue https://github.com/ceskaexpedice/ceska-digitalni-knihovna/issues/48 --- .../incad/kramerius/rest/oai/MetadataExport.java | 9 --------- .../cz/incad/kramerius/rest/oai/OAIVerb.java | 16 +++++++++++++--- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java index 220f330842..e3482e74ab 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java @@ -18,36 +18,27 @@ import java.io.IOException; import java.io.InputStream; -import java.net.URL; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; -import javax.inject.Named; import javax.inject.Provider; import javax.servlet.http.HttpServletRequest; -import javax.ws.rs.core.Response; import javax.xml.parsers.ParserConfigurationException; -import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.xml.sax.SAXException; -import com.google.inject.Inject; import com.sun.jersey.api.client.Client; -import cz.incad.kramerius.FedoraAccess; import cz.incad.kramerius.FedoraNamespaces; import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.fedora.om.RepositoryException; import cz.incad.kramerius.fedora.utils.CDKUtils; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.ProxyItemHandler; -import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.ProxyItemHandler.RequestMethodName; -import cz.incad.kramerius.rest.oai.exceptions.OAIException; import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.ApplicationURL; import cz.incad.kramerius.utils.IPAddressUtils; diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java index ec49699e10..9f57a1c05c 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java @@ -263,8 +263,13 @@ public void perform(Provider userProvider,Provider clientProvider, // Instances instances, HttpServletRequest request, Document owningDocument, String oaiIdentifier, MetadataExport export, OAISet set Element metadata = doc.createElement("metadata"); - metadata.appendChild(oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,selectedSet)); - + Element metadataElm = oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,selectedSet); + if (metadataElm != null) { + metadata.appendChild(metadataElm); + } else { + record.setAttribute("deleted","true"); + } + record.appendChild(header); record.appendChild(metadata); @@ -413,7 +418,12 @@ public void perform(Provider userProvider,Provider clientProvider, Element metadata = doc.createElement("metadata"); - metadata.appendChild(oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,null)); + Element metadataElm = oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,null); + if (metadataElm != null) { + metadata.appendChild(metadataElm); + } else { + record.setAttribute("deleted", "true"); + } record.appendChild(header); record.appendChild(metadata); From 1559e0085fa98b811ea01277b0c95ef9548687ef Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 13 Feb 2024 07:01:33 +0100 Subject: [PATCH 105/235] DBVersion initialization --- .../kramerius/security/database/SecurityDbInitializer.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/common/src/main/java/cz/incad/kramerius/security/database/SecurityDbInitializer.java b/shared/common/src/main/java/cz/incad/kramerius/security/database/SecurityDbInitializer.java index 28f270c130..92e5a11910 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/security/database/SecurityDbInitializer.java +++ b/shared/common/src/main/java/cz/incad/kramerius/security/database/SecurityDbInitializer.java @@ -328,7 +328,7 @@ public static void initDatabase(Connection connection, VersionService versionSer createNewActions(connection); - } else if (versionCondition(v, ">", "6.6.2")) { + } else if (versionCondition(v, ">", "6.6.2") && versionCondition(v, "<", "7.0.2")) { // labels table makeSureThatLabelsTable(connection); From 81a3f1c68f12ac238c3b3c2b289782e2782ea044 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 14 Apr 2024 10:07:53 +0200 Subject: [PATCH 106/235] Kubernetes fix --- .../cz/incad/kramerius/services/KubernetesProcessImpl.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java index 80487b65f8..0363a23467 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java @@ -14,6 +14,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.net.URL; import java.util.HashMap; import java.util.Map; import java.util.TimeZone; @@ -48,8 +49,11 @@ public static void main(String[] args) throws IOException, MigrateSolrIndexExcep if (env.containsKey(CONFIG_SOURCE) || args.length > 0) { String configSource = env.containsKey(CONFIG_SOURCE) ? env.get(CONFIG_SOURCE) : args[0]; InputStream stream = KubernetesProcessImpl.class.getResourceAsStream(configSource); + if (configSource.trim().startsWith("file:///")) { + URL fileUrl = new URL(configSource); + stream = fileUrl.openStream(); + } if (stream != null) { - Map iteration = new HashMap<>(); prefixVariables(env, iteration, ITERATION_PREFIX); From fabcd596a2b7285d204065bb86d2a19b3ac52f20 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 14 Apr 2024 10:59:30 +0200 Subject: [PATCH 107/235] Changes --- .../services/ParallelProcessImpl.java | 9 ++++---- .../cz/incad/kramerius/services/Worker.java | 6 +++-- .../kramerius/services/WorkerFactory.java | 2 +- .../kramerius/services/WorkerFinisher.java | 6 +++++ .../workers/checkexists/ExistsWorker.java | 2 +- .../checkexists/ExistsWorkerFactory.java | 2 +- .../workers/checkindex/CheckIndexWorker.java | 2 +- .../checkindex/CheckIndexWorkerFactory.java | 2 +- .../workers/nullworker/NullWorker.java | 2 +- .../workers/nullworker/NullWorkerFactory.java | 2 +- .../replicate/AbstractReplicateWorker.java | 5 +++-- .../workers/replicate/ReplicateFinisher.java | 22 +++++++++++++++++-- .../copy/CopyReplicateSolrWorkerFactory.java | 4 ++-- .../replicate/copy/CopyReplicateWorker.java | 20 +++++++++++++---- .../updatefield/UpdateFieldWorker.java | 5 +++-- .../updatefield/UpdateFieldWorkerFactory.java | 4 ++-- .../workers/update_dep/UpdateWorker.java | 5 +++-- .../update_dep/UpdateWorkerFactory.java | 4 ++-- .../updateocr_dep/UpdateOCRWorker.java | 2 +- ...UpdateOCRFromCDKEndpointWorkerFactory.java | 2 +- .../UpdateOCRFromSolrWorkerFactory.java | 2 +- .../configurations/defult_k5-update.xml | 2 +- .../replicate/configurations/default_k5.xml | 3 +++ .../replicate/configurations/default_k7.xml | 6 ++++- 24 files changed, 85 insertions(+), 36 deletions(-) diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java index 878e2a5e3e..f4572882b1 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java @@ -186,7 +186,6 @@ private void startWorkers(List worksWhasHasToBeDone, String workingtime) while (true) { if (!isInWorkingTime( startTime, endTime)) { - //LOGGER.info(String.format("No harvesting - waiting for end of interval %s", workingtime)); waitUntilStartWorkingTime(startTime, endTime); } else { break; @@ -262,11 +261,14 @@ public void migrate(File config) throws MigrateSolrIndexException, IllegalAccess }, ()-> { finishRestWorkers(worksWhatHasToBeDone,workingTime.get()); }); + } catch (Exception e) { + this.finisher.exceptionDuringCrawl(e); LOGGER.log(Level.SEVERE, e.getMessage(), e); throw new MigrateSolrIndexException(e); } finally { - if (finisher != null) this.finisher.finish(); + // stop process + if (finisher != null) this.finisher.finish(); } } @@ -275,7 +277,6 @@ private void initialize(Element iteration) { this.threads = threadsElm != null ? Integer.parseInt(threadsElm.getTextContent()) : 2; } - // musi zustat tady private void addNewWorkToWorkers(ProcessIterator processIterator, List worksWhatHasToBeDone, List identifiers, String workingtime) { try { worksWhatHasToBeDone.add(createWorker(processIterator, this.workerElem, identifiers)); @@ -292,7 +293,7 @@ private void addNewWorkToWorkers(ProcessIterator processIterator, List w private Worker createWorker(ProcessIterator iteratorInstance, Element workerElm, List identifiers) { try { - return this.workerFactory.createWorker(this.sourceName, iteratorInstance, workerElm, this.client,identifiers); + return this.workerFactory.createWorker(this.sourceName, iteratorInstance, workerElm, this.client,identifiers, this.finisher); } catch ( IllegalStateException e ) { throw new RuntimeException("Cannot create worker instance "+e.getMessage()); } diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/Worker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/Worker.java index 97d2adfa67..c7cfaca5bf 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/Worker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/Worker.java @@ -41,7 +41,8 @@ public abstract class Worker implements Runnable { protected String user; protected String pass; - + + protected WorkerFinisher finisher; static AtomicBoolean _LOGGER_INITIALIZED = new AtomicBoolean(false); static ReentrantLock _LOCK = new ReentrantLock(); @@ -73,8 +74,9 @@ private static void logConfigure(String logFileName, String loggerName) { - public Worker(String sourceName, Element workerElm, Client client, List items) { + public Worker(String sourceName, Element workerElm, Client client, List items, WorkerFinisher finisher) { super(); + this.finisher = finisher; this.client = client; this.itemsToBeProcessed = items; this.pidsToBeProcessed = items.stream().map(IterationItem::getPid).collect(Collectors.toList()); diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/WorkerFactory.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/WorkerFactory.java index b482525a40..c00c3877fd 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/WorkerFactory.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/WorkerFactory.java @@ -14,7 +14,7 @@ public abstract class WorkerFactory { public static final Logger LOGGER = Logger.getLogger(WorkerFactory.class.getName()); - public abstract Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element worker, Client client, List pids); + public abstract Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element worker, Client client, List pids, WorkerFinisher finisher); public abstract WorkerFinisher createFinisher(String timestampUrl, Element worker, Client client); diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/WorkerFinisher.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/WorkerFinisher.java index 1d96416730..0607657bbe 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/WorkerFinisher.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/WorkerFinisher.java @@ -31,6 +31,12 @@ public WorkerFinisher(String timestampUrl, Element workerElm, Client client) { } + + public void exceptionDuringCrawl(Exception ex) {} + + //public void finishSuccessCrawl() {} + + // inform about finish crawl public abstract void finish(); public CyclicBarrier getBarrier() { diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkexists/ExistsWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkexists/ExistsWorker.java index 19985afa40..605fed1f27 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkexists/ExistsWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkexists/ExistsWorker.java @@ -41,7 +41,7 @@ public static enum LogExistsCondition { private LogExistsCondition typeOfLogging = LogExistsCondition.BOTH; public ExistsWorker(String sourceName, Element workerElm, Client client, List items, Map cols) { - super(sourceName, workerElm, client, items); + super(sourceName,workerElm, client, items, null); this.collections = cols; diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkexists/ExistsWorkerFactory.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkexists/ExistsWorkerFactory.java index 7a617acb29..8a0a91d109 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkexists/ExistsWorkerFactory.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkexists/ExistsWorkerFactory.java @@ -28,7 +28,7 @@ public WorkerFinisher createFinisher(String timestampUrl, Element worker, Client } @Override - public Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element worker, Client client, List items) { + public Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element worker, Client client, List items, WorkerFinisher finisher) { Element requestElm = XMLUtils.findElement(worker, "request"); if (requestElm == null) throw new IllegalStateException("cannot find element request"); Element localKrameriusElm = XMLUtils.findElement(requestElm, "local.kramerius"); diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkindex/CheckIndexWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkindex/CheckIndexWorker.java index 7b6b3fd22b..3c19fcd898 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkindex/CheckIndexWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkindex/CheckIndexWorker.java @@ -33,7 +33,7 @@ public class CheckIndexWorker extends Worker { private String checkingIndexType; public CheckIndexWorker(String sourceName, Element workerElm, Client client, List pids) { - super(sourceName, workerElm, client, pids); + super(sourceName,workerElm, client, pids, null); Element destinationElm = XMLUtils.findElement(workerElm, "destination"); if (destinationElm != null) { diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkindex/CheckIndexWorkerFactory.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkindex/CheckIndexWorkerFactory.java index 2c93772279..c938d01d31 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkindex/CheckIndexWorkerFactory.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/checkindex/CheckIndexWorkerFactory.java @@ -22,7 +22,7 @@ public WorkerFinisher createFinisher(String timestampUrl, Element worker, Client } @Override - public Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element base, Client client, List items) { + public Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element base, Client client, List items, WorkerFinisher finisher) { return new CheckIndexWorker(sourceName,base, client, items); } } diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/nullworker/NullWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/nullworker/NullWorker.java index 7b88c61894..8476ef764b 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/nullworker/NullWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/nullworker/NullWorker.java @@ -23,7 +23,7 @@ public class NullWorker extends Worker { public NullWorker(String sourceName, Element workerElm, Client client, List pids) { - super(sourceName, workerElm, client, pids); + super(sourceName,workerElm, client, pids, null); } diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/nullworker/NullWorkerFactory.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/nullworker/NullWorkerFactory.java index b0bb66ef3d..726053de2a 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/nullworker/NullWorkerFactory.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/nullworker/NullWorkerFactory.java @@ -17,7 +17,7 @@ public class NullWorkerFactory extends WorkerFactory { public static int COUNTER = 0; @Override - public Worker createWorker(String name, ProcessIterator iteratorInstance, Element worker, Client client, List items) { + public Worker createWorker(String name, ProcessIterator iteratorInstance, Element worker, Client client, List items, WorkerFinisher finisher) { return new NullWorker(name, worker, client, items); } diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java index 22abcc02ec..d5cd89b638 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java @@ -17,6 +17,7 @@ import com.sun.jersey.api.client.Client; import cz.incad.kramerius.services.Worker; +import cz.incad.kramerius.services.WorkerFinisher; import cz.incad.kramerius.services.iterators.IterationItem; import cz.incad.kramerius.services.transform.SourceToDestTransform; import cz.incad.kramerius.services.utils.SolrUtils; @@ -50,8 +51,8 @@ public abstract class AbstractReplicateWorker extends Worker { protected String checkEndpoint = null; protected SourceToDestTransform transform; - public AbstractReplicateWorker(String sourceName, Element workerElm, Client client, List items) { - super(sourceName, workerElm, client, items); + public AbstractReplicateWorker(String sourceName, Element workerElm, Client client, List items, WorkerFinisher finisher) { + super(sourceName, workerElm, client, items, finisher); } // zjistuje, ziskava vsechny pidy, ktere jsou naindexovane diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateFinisher.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateFinisher.java index 794022d99c..b41e263151 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateFinisher.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateFinisher.java @@ -9,9 +9,13 @@ import cz.incad.kramerius.utils.StringUtils; import cz.incad.kramerius.utils.XMLUtils; +import org.apache.commons.collections4.list.FixedSizeList; import org.json.JSONObject; import org.w3c.dom.Element; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.logging.Logger; @@ -38,6 +42,11 @@ public class ReplicateFinisher extends WorkerFinisher { public static AtomicInteger NOT_INDEXED_COMPOSITEID = new AtomicInteger(0); public static AtomicInteger NOT_INDEXED_SKIPPED = new AtomicInteger(0); + // uknown exception during crawl + public static final int EXCEPTION_DURING_CRAWL_LIMIT = 10000; + public static List EXCEPTION_DURING_CRAWL = new ArrayList<>(); + + long start = System.currentTimeMillis(); protected String typeOfCrawl; @@ -67,13 +76,22 @@ private JSONObject storeTimestamp() { LOGGER.info(String.format("[" + Thread.currentThread().getName() + "] url %s", timestampUrl)); WebResource r = client.resource(timestampUrl); String t = r.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).entity(jsonObject.toString()).put(String.class); - return new JSONObject(t); } + + @Override + public void exceptionDuringCrawl(Exception ex) { + if (EXCEPTION_DURING_CRAWL.size() < EXCEPTION_DURING_CRAWL_LIMIT) { + EXCEPTION_DURING_CRAWL.add(ex); + LOGGER.info("Exception during crawl :"+EXCEPTION_DURING_CRAWL); + } + } + + @Override public void finish() { - if (StringUtils.isAnyString(timestampUrl)) { + if (StringUtils.isAnyString(timestampUrl) && EXCEPTION_DURING_CRAWL.isEmpty()) { storeTimestamp(); } SolrUtils.commit(this.client, this.destinationUrl); diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateSolrWorkerFactory.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateSolrWorkerFactory.java index 0125e29882..4ffe180ef9 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateSolrWorkerFactory.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateSolrWorkerFactory.java @@ -19,8 +19,8 @@ public class CopyReplicateSolrWorkerFactory extends WorkerFactory { @Override - public Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element base, Client client, List items) { - return new CopyReplicateWorker(sourceName, base, client, items); + public Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element base, Client client, List items, WorkerFinisher finisher) { + return new CopyReplicateWorker(sourceName, base, client, items, finisher); } @Override diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java index 1235f5d632..461c9b42e3 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java @@ -4,6 +4,7 @@ import cz.incad.kramerius.service.MigrateSolrIndexException; import cz.incad.kramerius.services.SupportedLibraries; import cz.incad.kramerius.services.Worker; +import cz.incad.kramerius.services.WorkerFinisher; import cz.incad.kramerius.services.iterators.IterationItem; import cz.incad.kramerius.services.utils.SolrUtils; import cz.incad.kramerius.services.workers.replicate.AbstractReplicateWorker; @@ -39,13 +40,12 @@ */ public class CopyReplicateWorker extends AbstractReplicateWorker { - //public static List LIBRARIES = Arrays.asList("nkp","blbec_k5"); public static SupportedLibraries supportedLibraries = new SupportedLibraries(); public static Logger LOGGER = Logger.getLogger(CopyReplicateWorker.class.getName()); - public CopyReplicateWorker(String sourceName, Element workerElm, Client client, List pids) { - super(sourceName, workerElm, client, pids); + public CopyReplicateWorker(String sourceName, Element workerElm, Client client, List pids, WorkerFinisher finisher) { + super(sourceName, workerElm, client, pids, finisher); config(workerElm); } @@ -334,22 +334,34 @@ public boolean acceptElement(Element element) { onUpdateEvent(addDocument); ReplicateFinisher.UPDATED.addAndGet(XMLUtils.getElements(addDocument).size()); String s = SolrUtils.sendToDest(this.destinationUrl, this.client, destBatch); - LOGGER.info(s); + //LOGGER.info(s); } else { LOGGER.info("No update element "); } } } catch (ParserConfigurationException e) { + LOGGER.log(Level.SEVERE,"Informing about exception"); + finisher.exceptionDuringCrawl(e); LOGGER.log(Level.SEVERE,e.getMessage(),e); } catch (SAXException e) { + LOGGER.log(Level.SEVERE,"Informing about exception"); + finisher.exceptionDuringCrawl(e); LOGGER.log(Level.SEVERE,e.getMessage(),e); } catch (IOException e) { + LOGGER.log(Level.SEVERE,"Informing about exception"); + finisher.exceptionDuringCrawl(e); LOGGER.log(Level.SEVERE,e.getMessage(),e); } catch (MigrateSolrIndexException e) { + LOGGER.log(Level.SEVERE,"Informing about exception"); + finisher.exceptionDuringCrawl(e); LOGGER.log(Level.SEVERE,e.getMessage(),e); } } + } catch(Exception ex) { + LOGGER.log(Level.SEVERE,"Informing about exception"); + finisher.exceptionDuringCrawl(ex); + LOGGER.log(Level.SEVERE,ex.getMessage(),ex); } finally { try { this.barrier.await(); diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/updatefield/UpdateFieldWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/updatefield/UpdateFieldWorker.java index a5a2ccc656..e2b9f0f508 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/updatefield/UpdateFieldWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/updatefield/UpdateFieldWorker.java @@ -19,6 +19,7 @@ import com.sun.jersey.api.client.Client; +import cz.incad.kramerius.services.WorkerFinisher; import cz.incad.kramerius.services.iterators.IterationItem; import cz.incad.kramerius.services.utils.SolrUtils; import cz.incad.kramerius.services.workers.replicate.AbstractReplicateWorker; @@ -31,8 +32,8 @@ public class UpdateFieldWorker extends AbstractReplicateWorker { private String updateField = null; private String updateOperation = null; - public UpdateFieldWorker(String sourceName, Element worker, Client client, List items) { - super(sourceName, worker, client, items); + public UpdateFieldWorker(String sourceName, Element worker, Client client, List items, WorkerFinisher finisher) { + super(sourceName, worker, client, items, finisher); config(workerElm); diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/updatefield/UpdateFieldWorkerFactory.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/updatefield/UpdateFieldWorkerFactory.java index 4e35bcefb3..35e8562eb7 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/updatefield/UpdateFieldWorkerFactory.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/updatefield/UpdateFieldWorkerFactory.java @@ -19,8 +19,8 @@ public class UpdateFieldWorkerFactory extends WorkerFactory { @Override - public Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element base, Client client, List items) { - return new UpdateFieldWorker(sourceName,base, client, items); + public Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element base, Client client, List items, WorkerFinisher finisher) { + return new UpdateFieldWorker(sourceName,base, client, items, finisher); } @Override diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/update_dep/UpdateWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/update_dep/UpdateWorker.java index 8262b5206d..e3503021d1 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/update_dep/UpdateWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/update_dep/UpdateWorker.java @@ -2,6 +2,7 @@ import com.sun.jersey.api.client.Client; import cz.incad.kramerius.services.Worker; +import cz.incad.kramerius.services.WorkerFinisher; import cz.incad.kramerius.services.iterators.IterationItem; import cz.incad.kramerius.services.utils.SolrUtils; import cz.incad.kramerius.utils.XMLUtils; @@ -23,8 +24,8 @@ public class UpdateWorker extends Worker { private List updateElements = new ArrayList<>(); - public UpdateWorker(String sourceName, Element workerElm, Client client, List pids) { - super(sourceName, workerElm, client, pids); + public UpdateWorker(String sourceName, Element workerElm, Client client, List pids, WorkerFinisher finisher) { + super(sourceName, workerElm, client, pids, finisher); Element destinationElm = XMLUtils.findElement(workerElm, "destination"); if (destinationElm != null) { Element updateFieldElement = XMLUtils.findElement(destinationElm, "update.dest.field"); diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/update_dep/UpdateWorkerFactory.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/update_dep/UpdateWorkerFactory.java index e44dfebfc5..19cd4cbce6 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/update_dep/UpdateWorkerFactory.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/update_dep/UpdateWorkerFactory.java @@ -20,7 +20,7 @@ public WorkerFinisher createFinisher(String timestampUrl, Element worker, Clien } @Override - public Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element worker, Client client, List items) { - return new UpdateWorker(sourceName, worker, client, items); + public Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element worker, Client client, List items, WorkerFinisher finisher) { + return new UpdateWorker(sourceName, worker, client, items, finisher); } } diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/updateocr_dep/UpdateOCRWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/updateocr_dep/UpdateOCRWorker.java index 03bd8b508f..00dadafd12 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/updateocr_dep/UpdateOCRWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/updateocr_dep/UpdateOCRWorker.java @@ -26,7 +26,7 @@ public abstract class UpdateOCRWorker extends Worker { private String pass; public UpdateOCRWorker(String sourceName, Element worker, Client client, List items) { - super(sourceName, worker, client, items); + super(sourceName, worker, client, items,null); Element request = XMLUtils.findElement(workerElm, "request"); if (request != null) { diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/updateocrcdk_dep/UpdateOCRFromCDKEndpointWorkerFactory.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/updateocrcdk_dep/UpdateOCRFromCDKEndpointWorkerFactory.java index 4531005f2f..ad25985648 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/updateocrcdk_dep/UpdateOCRFromCDKEndpointWorkerFactory.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/updateocrcdk_dep/UpdateOCRFromCDKEndpointWorkerFactory.java @@ -22,7 +22,7 @@ public WorkerFinisher createFinisher(String timestampUrl, Element worker, Client } @Override - public Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element worker, Client client, List items) { + public Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element worker, Client client, List items, WorkerFinisher finihser) { return new UpdateOCRFromCDKEndpointWorker(sourceName, worker, client, items); } } diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/updateocrsolr_dep/UpdateOCRFromSolrWorkerFactory.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/updateocrsolr_dep/UpdateOCRFromSolrWorkerFactory.java index 72b8b49608..fb2750d4d0 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/updateocrsolr_dep/UpdateOCRFromSolrWorkerFactory.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/updateocrsolr_dep/UpdateOCRFromSolrWorkerFactory.java @@ -15,7 +15,7 @@ public class UpdateOCRFromSolrWorkerFactory extends WorkerFactory { @Override - public Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element worker, Client client, List pids) { + public Worker createWorker(String sourceName, ProcessIterator iteratorInstance, Element worker, Client client, List pids, WorkerFinisher finisher) { return new UpdateOCRFromSolrWorker(sourceName, worker, client, pids); } diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/defult_k5-update.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/defult_k5-update.xml index b49cb3c163..d0be3943de 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/defult_k5-update.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/periodicupdates/configurations/defult_k5-update.xml @@ -76,7 +76,7 @@ - PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model dc.title search_autor facet_autor $iteration.dl$ diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml index 2b347c976b..9f7b0a3bf4 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml @@ -6,6 +6,7 @@ full $iteration.workingtime$ + $timestamp.url$ @@ -20,6 +21,8 @@ $iteration.user$ $iteration.pass$ + $iteration.fquery$ + modified_date diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml index ee02a4eb2a..047ef2434e 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k7.xml @@ -4,6 +4,10 @@ $iteration.dl$ $iteration.dl$-full full + + $iteration.workingtime$ + $timestamp.url$ + @@ -18,7 +22,7 @@ - 2 + 1 From 25c572b446fe56b7db501f9f9627a1e3b1cf4560 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 21 Apr 2024 12:34:47 +0200 Subject: [PATCH 108/235] Reharvest pids --- .../services/ParallelProcessImpl.java | 8 +- .../transform/K7SourceToDestTransform.java | 2 + .../replicate/AbstractReplicateWorker.java | 1 + .../replicate/copy/CopyReplicateWorker.java | 111 +++++-- .../replicate/configurations/default_k5.xml | 2 +- ...lelProcessImplTest_REPLICATE_Specific.java | 292 ++++++++++++++++-- .../services/config.replicate.mlp.xml | 23 +- .../config.replicate.mlp_k7_solrcloud.xml | 11 +- .../cz/incad/kramerius/ReharvestPids.java | 161 ++++++---- 9 files changed, 480 insertions(+), 131 deletions(-) diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java index f4572882b1..d1613726e4 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java @@ -9,6 +9,7 @@ import cz.incad.kramerius.services.iterators.ProcessIterator; import cz.incad.kramerius.services.iterators.ProcessIteratorFactory; import cz.incad.kramerius.timestamps.TimestampStore; +import cz.incad.kramerius.utils.StringUtils; import cz.incad.kramerius.utils.XMLUtils; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -212,7 +213,7 @@ public void migrate(File config) throws MigrateSolrIndexException, IllegalAccess Element timestampElm = XMLUtils.findElement(document.getDocumentElement(),"timestamp"); String timestamp = null; if (timestampElm != null) { - timestamp = timestampElm.getTextContent(); + timestamp = StringUtils.isAnyString(timestampElm.getTextContent().trim()) ? timestampElm.getTextContent() : null ; } Element workingTimeElm = XMLUtils.findElement(document.getDocumentElement(),"workingtime"); @@ -230,6 +231,8 @@ public void migrate(File config) throws MigrateSolrIndexException, IllegalAccess timestamp = timestamp+(timestamp.endsWith("/") ? "" : "/")+String.format("%s/timestamp",this.sourceName); } } + } else { + throw new IllegalStateException("expecting source name !! "); } // Iterator factory @@ -239,7 +242,8 @@ public void migrate(File config) throws MigrateSolrIndexException, IllegalAccess // Iterator instance Element iterationElm = XMLUtils.findElement(document.getDocumentElement(), "iteration"); - this.iterator =processIteratorFactory.createProcessIterator(timestamp, iterationElm, this.client); + + this.iterator =processIteratorFactory.createProcessIterator(timestamp, iterationElm, this.client); Element workerFactory = XMLUtils.findElement(document.getDocumentElement(),"workerFactory"); String workerClass = workerFactory.getAttribute("class"); diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java index 53850fa0b1..4f5e8f5891 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java @@ -403,12 +403,14 @@ public boolean acceptElement(Element element) { private void translateAccessibilityToLicense(String model, String accessibility, Document destDocument, Element destDocElem, CopyReplicateConsumer consumer) { + // Translate accessibility to license String license = "public".equals(accessibility) ? "public" : "onsite"; List markContainsLicenses = Arrays.asList("monographunit","periodicalvolume"); List licenses = Arrays.asList("monograph","monographunit", "periodicalvolume","map","sheetmusic"); List licenses_of_ancestors = Arrays.asList("page","supplement","periodicalitem"); + if (licenses.contains(model)) { field(destDocument, destDocElem,license, "licenses",consumer); } else if (licenses_of_ancestors.contains(model)) { diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java index d5cd89b638..3901e8bab7 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/AbstractReplicateWorker.java @@ -282,6 +282,7 @@ public Element fetchDocumentFromRemoteSOLR(Client client, List pids, Str }); String query = "?q=" + idIdentifier + ":(" + URLEncoder.encode(reduce, "UTF-8") + ")&fl=" + URLEncoder.encode(fieldlist, "UTF-8") + "&wt=xml&rows=" + pids.size(); + return SolrUtils.executeQuery(client, this.requestUrl, query, this.user, this.pass); } } diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java index 461c9b42e3..dfb8f3a185 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/copy/CopyReplicateWorker.java @@ -21,9 +21,13 @@ import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; import java.io.IOException; +import java.io.StringWriter; +import java.time.Instant; +import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -81,6 +85,20 @@ public ModifyFieldResult modifyField(Element field) { @Override public void changeDocument(String rootPid, String pid,Element doc) { + List indexed = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String attribute = element.getAttribute("name"); + return "indexed".equals(attribute); + } + }).stream().collect(Collectors.toList()); + + + if (indexed.size() > 0) { + Instant instant = new Date().toInstant(); + indexed.get(0).setTextContent(DateTimeFormatter.ISO_INSTANT.format(instant)); + } + List licensesOfAncestors = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { @Override @@ -134,6 +152,8 @@ public boolean acceptElement(Element element) { doc.appendChild(cdkLicenses); } + + } }); @@ -192,13 +212,20 @@ public ModifyFieldResult modifyField(Element field) { List deleteFields = Arrays.asList(); List addValues = Arrays.asList( - // toto musi byt pocitane pole "licenses", "licenses_of_ancestors", "contains_licenses", "in_collections", - "in_collections.direct"); + "in_collections.direct", + + "titles.search", + "authors", + "authors.search", + "authors.facet", + + "cdk.k5.license.translated", + "cdk.licenses"); String name = field.getAttribute("name"); if (deleteFields.contains(name)) { @@ -220,18 +247,30 @@ public ModifyFieldResult modifyField(Element field) { public void changeDocument(String root, String pid, Element doc) { // setreseni dat cdk.licenses + //2023-06-26T12:56:07.400Z + + + Instant instant = new Date().toInstant(); + //DateTimeFormatter.ISO_INSTANT.format(instant); + Element fieldDate = doc.getOwnerDocument().createElement("field"); + fieldDate.setAttribute("name", "indexed"); + fieldDate.setAttribute("update", "set"); + fieldDate.setTextContent(DateTimeFormatter.ISO_INSTANT.format(instant)); + doc.appendChild(fieldDate); + + List> comparingFields = Arrays.asList( Pair.of("licenses", "cdk.licenses"), Pair.of("contains_licenses", "cdk.contains_licenses"), Pair.of("licenses_of_ancestors", "cdk.licenses_of_ancestors") ); + //System.out.println("Docs for pid "+pid); Map cdkDoc = docs.get(pid); for (Pair cpField : comparingFields) { String sourceField = cpField.getLeft(); String specificCDKField = cpField.getRight(); - List newIndexedField = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { @Override public boolean acceptElement(Element element) { @@ -256,15 +295,6 @@ public boolean acceptElement(Element element) { // add new indexed values newList.addAll(newCDKValues); -// // collect all libraries -// Set libraries = newList.stream().map(it-> { -// int idx = it.indexOf("_"); -// if (idx > 0) { -// return it.substring(0, idx); -// } else { -// return it; -// } -// }).collect(Collectors.toSet()); for (Element nIF : newIndexedField) { doc.removeChild(nIF); @@ -272,30 +302,48 @@ public boolean acceptElement(Element element) { Set tempSet = new HashSet<>(); Document document = doc.getOwnerDocument(); - newList.stream().forEach(lic-> { + if (newList.size() > 0) { + newList.stream().forEach(lic-> { + + Element cdkSpecific = document.createElement("field"); + cdkSpecific.setAttribute("name", specificCDKField); + cdkSpecific.setAttribute("update", "set"); + cdkSpecific.setTextContent(lic); + doc.appendChild(cdkSpecific); + + Pair divided = supportedLibraries.divideLibraryAndLicense(lic); + if (divided != null) { + + String rv = divided.getRight(); + if (!tempSet.contains(rv)) { + Element changedField = document.createElement("field"); + changedField.setAttribute("name", sourceField); + changedField.setAttribute("update", "set"); + + changedField.setTextContent(rv); + doc.appendChild(changedField); + + tempSet.add(rv); + } + } + }); + + } else { + Element cdkSpecific = document.createElement("field"); cdkSpecific.setAttribute("name", specificCDKField); cdkSpecific.setAttribute("update", "set"); - cdkSpecific.setTextContent(lic); + cdkSpecific.setAttribute("null", "true"); doc.appendChild(cdkSpecific); + + Element changedField = document.createElement("field"); + changedField.setAttribute("name", sourceField); + changedField.setAttribute("update", "set"); + changedField.setAttribute("null", "true"); + doc.appendChild(changedField); - Pair divided = supportedLibraries.divideLibraryAndLicense(lic); - if (divided != null) { - - String rv = divided.getRight(); - if (!tempSet.contains(rv)) { - Element changedField = document.createElement("field"); - changedField.setAttribute("name", sourceField); - changedField.setAttribute("update", "set"); - - changedField.setTextContent(rv); - doc.appendChild(changedField); - - tempSet.add(rv); - } - } - }); + } } } } @@ -306,7 +354,7 @@ public boolean acceptElement(Element element) { Element doc = db.createElement("doc"); Element field = db.createElement("field"); if (compositeId) { - String compositeId = pair.get("compositeId").toString(); + //String compositeId = pair.get("compositeId").toString(); String root = pair.get(transform.getField(rootOfComposite)).toString(); String child = pair.get(transform.getField(childOfComposite)).toString(); @@ -333,6 +381,7 @@ public boolean acceptElement(Element element) { Element addDocument = destBatch.getDocumentElement(); onUpdateEvent(addDocument); ReplicateFinisher.UPDATED.addAndGet(XMLUtils.getElements(addDocument).size()); + String s = SolrUtils.sendToDest(this.destinationUrl, this.client, destBatch); //LOGGER.info(s); } else { diff --git a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml index 9f7b0a3bf4..ac4f70cd25 100644 --- a/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml +++ b/processes/cdkprocesses/src/main/resources/cz/incad/kramerius/services/workers/replicate/configurations/default_k5.xml @@ -80,7 +80,7 @@ - PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model dc.title search_autor facet_autor $iteration.dl$ diff --git a/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/ParallelProcessImplTest_REPLICATE_Specific.java b/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/ParallelProcessImplTest_REPLICATE_Specific.java index 87ea170298..8810ada7fd 100644 --- a/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/ParallelProcessImplTest_REPLICATE_Specific.java +++ b/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/ParallelProcessImplTest_REPLICATE_Specific.java @@ -3,6 +3,8 @@ import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; +import com.sun.mail.iap.ByteArray; + import cz.incad.kramerius.service.MigrateSolrIndexException; import cz.incad.kramerius.utils.XMLUtils; import junit.framework.Assert; @@ -19,10 +21,14 @@ import javax.ws.rs.core.MediaType; import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; + +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.InputStream; import java.io.StringReader; +import java.io.StringWriter; import java.net.URISyntaxException; import java.util.List; @@ -31,38 +37,154 @@ import static org.easymock.EasyMock.anyObject; import static org.xmlunit.assertj.XmlAssert.assertThat; -@Ignore +//@Ignore public class ParallelProcessImplTest_REPLICATE_Specific { /** K7 transform - KNAV error - duplicate values rels-exts.sort */ @Test - public void testSpecificCase_1() throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { + public void testSpecificCase_1() + throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, + ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { - InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("config.replicate.mlp_k7_solrcloud.xml"); + InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class + .getResourceAsStream("config.replicate.mlp_k7_solrcloud.xml"); String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); - InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("k7notexists_specific/_1/logfile_log"); + InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class + .getResourceAsStream("k7notexists_specific/_1/logfile_log"); String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); - File logFile = File.createTempFile("junit","logfile"); + File logFile = File.createTempFile("junit", "logfile"); FileUtils.write(logFile, logFileContent, "UTF-8"); String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); - File configurationFile = File.createTempFile("junit","conf"); + File configurationFile = File.createTempFile("junit", "conf"); FileUtils.write(configurationFile, configurationContent, "UTF-8"); + Client client = EasyMock.createMock(Client.class); + ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class).addMockedMethod("buildClient") + .createMock(); + + // --- 1 doc indexed --- + String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3A63bd79e0-d62a-40f1-a374-57e2d78827c9%22)&fl=pid+collection+cdk.leader+cdk.collection+cdk.licenses+cdk.licenses_of_ancestors+cdk.contains_licenses++root.pid+compositeId&wt=xml&rows=1"; + String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists_specific/_1/1.xml"), + "UTF-8"); + List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); + + String secondReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A63bd79e0-d62a-40f1-a374-57e2d78827c9%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels+contains-dnnt-labels&wt=xml&rows=1"; + String secondRespFetch = IOUtils + .toString(this.getClass().getResourceAsStream("k7notexists_specific/_1/1_fetch.xml"), "UTF-8"); + List mocksFromSecondFetchCall = webCallExpect(client, secondReqFetch, secondRespFetch); + // ---------- + + EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); + + // all updates in one check + WebResource updateResource = EasyMock.createMock(WebResource.class); + ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); + WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); + EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")) + .andReturn(updateResource).anyTimes(); + EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); + EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); + + // EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), + // EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); + // Capture captureSingleArgument = EasyMock.newCa + Capture firstArg = newCapture(); + EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))) + .andReturn(updateResponseBuilder).once(); + + EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); + EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); + EasyMock.expect(updateResponse.getEntityInputStream()) + .andDelegateTo(new ParallelProcessImplTest_REPLICATE.MockClientResponse()).anyTimes(); + + // final commit + WebResource commitResource = EasyMock.createMock(WebResource.class); + WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); + + EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")) + .andReturn(commitResource).anyTimes(); + EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); + EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); + EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))) + .andReturn(commitResourceBuilder).anyTimes(); + EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); + + EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, commitResource, + commitResourceBuilder); + + // check & fetch + mocksFromSecondCall.stream().forEach(obj -> { + EasyMock.replay(obj); + }); + mocksFromSecondFetchCall.stream().forEach(obj -> { + EasyMock.replay(obj); + }); + + process.setClient(client); + + // start whole process + process.migrate(configurationFile); + + Document document = XMLUtils.parseDocument(new StringReader(firstArg.getValue().toString())); + List docs = XMLUtils.getElements(document.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + return element.getNodeName().equals("doc"); + } + }); + + Assert.assertTrue(document.getDocumentElement() != null); + + docs.stream().forEach(doc -> { + List elements = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("field")) { + return element.getAttribute("name").equals("rels_ext_index.sort"); + } + return false; + } + }); + if (!elements.isEmpty()) { + Assert.assertTrue(elements.size() == 1); + } + }); + } + + @Test + public void testSpecificCase_2() + throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, + ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { + InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class + .getResourceAsStream("config.replicate.mlp_k7_solrcloud.xml"); + String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); + + InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class + .getResourceAsStream("k7exists_specific/_1/logfile_log"); + String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); + File logFile = File.createTempFile("junit", "logfile"); + FileUtils.write(logFile, logFileContent, "UTF-8"); + + String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); + File configurationFile = File.createTempFile("junit", "conf"); + FileUtils.write(configurationFile, configurationContent, "UTF-8"); Client client = EasyMock.createMock(Client.class); - ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class) - .addMockedMethod("buildClient") + ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class).addMockedMethod("buildClient") .createMock(); - // --- 1 doc indexed --- - String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3A63bd79e0-d62a-40f1-a374-57e2d78827c9%22)&fl=pid+collection+root.pid&wt=xml&rows=1"; - String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists_specific/_1/1.xml"), "UTF-8"); + // --- 1 doc indexed --- + String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3A63bd79e0-d62a-40f1-a374-57e2d78827c9%22)&fl=pid+collection+cdk.leader+cdk.collection+cdk.licenses+cdk.licenses_of_ancestors+cdk.contains_licenses++root.pid+compositeId&wt=xml&rows=1"; + String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("k7exists_specific/_1/1.xml"), + "UTF-8"); List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); - String secondReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A63bd79e0-d62a-40f1-a374-57e2d78827c9%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=1"; - String secondRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists_specific/_1/1_fetch.xml"), "UTF-8"); + String secondReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A63bd79e0-d62a-40f1-a374-57e2d78827c9%22)&fl=PID+root_pid+dnnt-labels+contains-dnnt-labels+dostupnost+fedora.model+dc.title+search_autor+facet_autor&wt=xml&rows=1"; + + String secondRespFetch = IOUtils + .toString(this.getClass().getResourceAsStream("k7exists_specific/_1/1_fetch_update.xml"), "UTF-8"); List mocksFromSecondFetchCall = webCallExpect(client, secondReqFetch, secondRespFetch); // ---------- @@ -72,38 +194,43 @@ public void testSpecificCase_1() throws IOException, IllegalAccessException, Ins WebResource updateResource = EasyMock.createMock(WebResource.class); ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")).andReturn(updateResource).anyTimes(); + EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")) + .andReturn(updateResource).anyTimes(); EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); - //EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); - //Capture captureSingleArgument = EasyMock.newCa + // EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), + // EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); + // Capture captureSingleArgument = EasyMock.newCa Capture firstArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); + EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))) + .andReturn(updateResponseBuilder).once(); EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); - EasyMock.expect(updateResponse.getEntityInputStream()).andDelegateTo(new ParallelProcessImplTest_REPLICATE.MockClientResponse()).anyTimes(); + EasyMock.expect(updateResponse.getEntityInputStream()) + .andDelegateTo(new ParallelProcessImplTest_REPLICATE.MockClientResponse()).anyTimes(); // final commit WebResource commitResource = EasyMock.createMock(WebResource.class); WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")).andReturn(commitResource).anyTimes(); + EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")) + .andReturn(commitResource).anyTimes(); EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(commitResourceBuilder).anyTimes(); + EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))) + .andReturn(commitResourceBuilder).anyTimes(); EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); - EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, - commitResource, + EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, commitResource, commitResourceBuilder); // check & fetch - mocksFromSecondCall.stream().forEach(obj-> { + mocksFromSecondCall.stream().forEach(obj -> { EasyMock.replay(obj); }); - mocksFromSecondFetchCall.stream().forEach(obj-> { + mocksFromSecondFetchCall.stream().forEach(obj -> { EasyMock.replay(obj); }); @@ -112,7 +239,7 @@ public void testSpecificCase_1() throws IOException, IllegalAccessException, Ins // start whole process process.migrate(configurationFile); - //System.out.println(firstArg.getValue()); + // System.out.println(firstArg.getValue()); Document document = XMLUtils.parseDocument(new StringReader(firstArg.getValue().toString())); List docs = XMLUtils.getElements(document.getDocumentElement(), new XMLUtils.ElementsFilter() { @@ -122,7 +249,7 @@ public boolean acceptElement(Element element) { } }); - docs.stream().forEach(doc-> { + docs.stream().forEach(doc -> { List elements = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { @Override public boolean acceptElement(Element element) { @@ -136,7 +263,120 @@ public boolean acceptElement(Element element) { Assert.assertTrue(elements.size() == 1); } }); + } + + @Test + public void testSpecificCase_3() + throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, + ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { + InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class + .getResourceAsStream("config.replicate.mlp_k7_solrcloud.xml"); + String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); + System.out.println(_configurationContent); + + InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class + .getResourceAsStream("k7exists_specific/_2/logfile_log"); + String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); + File logFile = File.createTempFile("junit", "logfile"); + FileUtils.write(logFile, logFileContent, "UTF-8"); + + String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); + File configurationFile = File.createTempFile("junit", "conf"); + FileUtils.write(configurationFile, configurationContent, "UTF-8"); + + Client client = EasyMock.createMock(Client.class); + ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class).addMockedMethod("buildClient") + .createMock(); + + // --- 1 doc indexed --- + // uuid:3ed67a55-3346-410e-a0e8-4dcf64a14e71/@1 + // http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3A029a053a-0bc8-47aa-bac7-d2dd453e0ae3%2F%401%22)&fl=pid+collection+cdk.leader+cdk.collection+cdk.licenses+cdk.licenses_of_ancestors+cdk.contains_licenses++root.pid+compositeId&wt=xml&rows=1 + String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3A029a053a-0bc8-47aa-bac7-d2dd453e0ae3%2F%401%22)&fl=pid+collection+cdk.leader+cdk.collection+cdk.licenses+cdk.licenses_of_ancestors+cdk.contains_licenses++root.pid+compositeId&wt=xml&rows=1"; + String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("k7exists_specific/_2/2.xml"), + "UTF-8"); + List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); + + // http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A029a053a-0bc8-47aa-bac7-d2dd453e0ae3%2F%401%22)&fl=PID+root_pid+dnnt-labels+contains-dnnt-labels+dostupnost+fedora.model+dc.title+search_autor+facet_autor&wt=xml&rows=1 + String secondReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A029a053a-0bc8-47aa-bac7-d2dd453e0ae3%2F%401%22)&fl=PID+root_pid+dnnt-labels+contains-dnnt-labels+dostupnost+fedora.model+dc.title+search_autor+facet_autor&wt=xml&rows=1"; + String secondRespFetch = IOUtils + .toString(this.getClass().getResourceAsStream("k7exists_specific/_2/2_fetch_update.xml"), "UTF-8"); + List mocksFromSecondFetchCall = webCallExpect(client, secondReqFetch, secondRespFetch); + // ---------- + + EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); + + // all updates in one check + WebResource updateResource = EasyMock.createMock(WebResource.class); + ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); + WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); + EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")) + .andReturn(updateResource).anyTimes(); + EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); + EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); + + // EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), + // EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); + // Capture captureSingleArgument = EasyMock.newCa + Capture firstArg = newCapture(); + EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))) + .andReturn(updateResponseBuilder).once(); + + EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); + EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); + EasyMock.expect(updateResponse.getEntityInputStream()) + .andDelegateTo(new ParallelProcessImplTest_REPLICATE.MockClientResponse()).anyTimes(); + + // final commit + WebResource commitResource = EasyMock.createMock(WebResource.class); + WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); + + EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")) + .andReturn(commitResource).anyTimes(); + EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); + EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); + EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))) + .andReturn(commitResourceBuilder).anyTimes(); + EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); + + EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, commitResource, + commitResourceBuilder); + + // check & fetch + mocksFromSecondCall.stream().forEach(obj -> { + EasyMock.replay(obj); + }); + mocksFromSecondFetchCall.stream().forEach(obj -> { + EasyMock.replay(obj); + }); + process.setClient(client); + + // start whole process + process.migrate(configurationFile); + + // System.out.println(firstArg.getValue()); + + Document document = XMLUtils.parseDocument(new StringReader(firstArg.getValue().toString())); + List docs = XMLUtils.getElements(document.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + return element.getNodeName().equals("doc"); + } + }); + docs.stream().forEach(doc -> { + List elements = XMLUtils.getElements(doc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("field")) { + return element.getAttribute("name").equals("rels_ext_index.sort"); + } + return false; + } + }); + if (!elements.isEmpty()) { + Assert.assertTrue(elements.size() == 1); + } + }); } } diff --git a/processes/cdkprocesses/src/test/resources/cz/incad/kramerius/services/config.replicate.mlp.xml b/processes/cdkprocesses/src/test/resources/cz/incad/kramerius/services/config.replicate.mlp.xml index e7f10284b5..9891e975e3 100644 --- a/processes/cdkprocesses/src/test/resources/cz/incad/kramerius/services/config.replicate.mlp.xml +++ b/processes/cdkprocesses/src/test/resources/cz/incad/kramerius/services/config.replicate.mlp.xml @@ -1,6 +1,9 @@ + mkp + mkp-full + @@ -22,15 +25,14 @@ http://10.19.6.10:8983/solr/k7_5 select - - - - vc:d4b466de-5435-4b76-bff7-2838bbae747b - http://kramerius4.mlp.cz/search - - + true + root_pid + PID + + K7 + @@ -49,10 +51,13 @@ - - added-collection + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + knav + diff --git a/processes/cdkprocesses/src/test/resources/cz/incad/kramerius/services/config.replicate.mlp_k7_solrcloud.xml b/processes/cdkprocesses/src/test/resources/cz/incad/kramerius/services/config.replicate.mlp_k7_solrcloud.xml index 18e7402497..873726f4ad 100644 --- a/processes/cdkprocesses/src/test/resources/cz/incad/kramerius/services/config.replicate.mlp_k7_solrcloud.xml +++ b/processes/cdkprocesses/src/test/resources/cz/incad/kramerius/services/config.replicate.mlp_k7_solrcloud.xml @@ -3,6 +3,14 @@ + + mlp + mlp-full + full + + + + %s @@ -61,8 +69,9 @@ + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model dc.title search_autor facet_autor - added-collection + knav diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java index 87f67fe7e6..35d91d0be9 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java @@ -4,6 +4,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.StringWriter; import java.net.URLEncoder; import java.nio.charset.Charset; import java.util.ArrayList; @@ -40,6 +41,8 @@ import cz.incad.kramerius.impl.SolrAccessImplNewIndex; import cz.incad.kramerius.resourceindex.ResourceIndexException; import cz.incad.kramerius.resourceindex.ResourceIndexModule; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.properties.DefaultPropertiesInstances; import cz.incad.kramerius.service.MigrateSolrIndexException; import cz.incad.kramerius.solr.SolrModule; import cz.incad.kramerius.statistics.NullStatisticsModule; @@ -80,85 +83,121 @@ public static void main(String[] args) throws IOException, SolrServerException, // //params from lp.st String authToken = args[argsIndex++]; String target = args[argsIndex++]; //auth token always second, but still suboptimal solution, best would be if it was outside the scope of this as if ProcessHelper.scheduleProcess() similarly to changing name (ProcessStarter) + String onlyShowConfiguration = null; + if (args.length>=3) { + onlyShowConfiguration = args[argsIndex++]; + } + List extractPids = extractPids(target); - for (String pid : extractPids) { reharvestPID(pid); } + for (String pid : extractPids) { reharvestPID(pid, onlyShowConfiguration); } } - private static void reharvestPID(String pid) + private static void reharvestPID(String pid, String onlyShowConfiguration) throws IOException, TransformerException, MigrateSolrIndexException, ParserConfigurationException { Injector injector = Guice.createInjector(new SolrModule(), new ResourceIndexModule(), new RepoModule(), new NullStatisticsModule(), new ResourceIndexModule()); SolrAccess searchIndex = injector.getInstance(Key.get(SolrAccessImplNewIndex.class)); //FIXME: hardcoded implementation Document document = searchIndex.getSolrDataByPid(pid); - int nf = numFound(document); - if (nf > 0) { - - String msg = String.format("reharvesting pid %s ", pid); - LOGGER.info(msg); - String leader = leader(document); - List collections = collections(document); - - ParallelProcessImpl parallelProcess = new ParallelProcessImpl(); - Document deleteByQuery = deleteRootPid(searchIndex, pid); - - String s = SolrUtils.sendToDest(getDestinationUpdateUrl(), parallelProcess.getClient(), deleteByQuery); - - List harvestFiles = new ArrayList<>(); + + if (document != null) { + // nalezen v solru + int nf = numFound(document); + if (nf > 0) { + + String msg = String.format("Reharvesting pid %s ", pid); + LOGGER.info(msg); - // leader must be first - collections.remove(leader); - collections.add(0, leader); - - for (String ac : collections) { - try { - String api = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + ac + ".api"); + String leader = leader(document); + List collections = collections(document); + + ParallelProcessImpl parallelProcess = new ParallelProcessImpl(); + Document deleteByQuery = deleteRootPid(searchIndex, pid); + if (isOnlyShowConfiguration(onlyShowConfiguration)) { + StringWriter writer = new StringWriter(); + XMLUtils.print(deleteByQuery, writer); + LOGGER.info("Delete by query "+writer.toString()); + } else { + String s = SolrUtils.sendToDest(getDestinationUpdateUrl(), parallelProcess.getClient(), deleteByQuery); + } + + collections.remove(leader); + collections.add(0, leader); + + reharvestPIDFromGivenCollections(pid, collections, onlyShowConfiguration); + } else { + DefaultPropertiesInstances props = new DefaultPropertiesInstances(); + List enabledInstances = props.enabledInstances(); + reharvestPIDFromGivenCollections(pid, enabledInstances.stream().map(OneInstance::getName).collect(Collectors.toList()), onlyShowConfiguration); + } + } + } - Pair iterationUrl = iterationUrl(ac); + + private static boolean isOnlyShowConfiguration(String onlyShowConfiguration) { + if (onlyShowConfiguration != null && ("onlyshowconfiguration".equals(onlyShowConfiguration.toLowerCase()) || "true".equals(onlyShowConfiguration))) { + return true; + } + return true; + } - Map iteration = new HashMap<>(); - iteration.put("url", iterationUrl.getKey()); - iteration.put("dl", ac); - iteration.put("fquery", fq(api, pid)); - - - if (iterationUrl.getRight()) { - String username = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + ac + ".username"); - String password = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + ac + ".pswd"); + private static void reharvestPIDFromGivenCollections(String pid, List collections, String onlyShowConfiguration) { + List harvestFiles = new ArrayList<>(); + for (String ac : collections) { + try { + String api = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + ac + ".api"); + if (api == null) { + LOGGER.warning(String.format("Skipping instance %s", ac)); + continue; + } + + + Pair iterationUrl = iterationUrl(ac); - iteration.put("user", username); - iteration.put("pass", password); - } - - // destination - Map destination = new HashMap<>(); - destination.put("url", KConfiguration.getInstance().getSolrSearchHost()); - - boolean publicEndpoint = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + ac + ".public") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + ac + ".public") : false; + Map iteration = new HashMap<>(); + iteration.put("url", iterationUrl.getKey()); + iteration.put("dl", ac); + iteration.put("fquery", fq(api, pid)); + + + if (iterationUrl.getRight()) { + String username = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + ac + ".username"); + String password = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + ac + ".pswd"); - String configuration = renderTemplate(publicEndpoint, api, iteration, destination); - File tmpFile = File.createTempFile(String.format("%s", ac), "reharvest"); - - Files.write(configuration.getBytes("UTF-8"), tmpFile); - harvestFiles.add(tmpFile); - - } catch (IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); + iteration.put("user", username); + iteration.put("pass", password); } + + // destination + Map destination = new HashMap<>(); + destination.put("url", KConfiguration.getInstance().getSolrSearchHost()); + + boolean publicEndpoint = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + ac + ".public") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + ac + ".public") : false; + + String configuration = renderTemplate(publicEndpoint, api, iteration, destination); + File tmpFile = File.createTempFile(String.format("%s", ac), "reharvest"); + + Files.write(configuration.getBytes("UTF-8"), tmpFile); + harvestFiles.add(tmpFile); + + } catch (IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); } - - for (File harvestFile : harvestFiles) { - try { + } + + for (File harvestFile : harvestFiles) { + try { + if (isOnlyShowConfiguration(onlyShowConfiguration)) { + String config = org.apache.commons.io.IOUtils.toString(new FileInputStream(harvestFile), "UTF-8"); + LOGGER.info(String.format("Configuration %s" ,config)); + } else { ParallelProcessImpl reharvest = new ParallelProcessImpl(); String config = org.apache.commons.io.IOUtils.toString(new FileInputStream(harvestFile), "UTF-8"); reharvest.migrate(harvestFile); - - } catch (IOException | MigrateSolrIndexException | IllegalAccessException | InstantiationException | ClassNotFoundException | NoSuchMethodException | ParserConfigurationException | SAXException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); } + + } catch (IOException | MigrateSolrIndexException | IllegalAccessException | InstantiationException | ClassNotFoundException | NoSuchMethodException | ParserConfigurationException | SAXException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); } - } else { - String msg = String.format("pid %s not found", pid); - LOGGER.warning( msg); } } @@ -205,7 +244,7 @@ public static Pair iterationUrl(String acronym) { String retval = channel + (channel.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/sync/solr"; return Pair.of(retval, false); } else { - String retval = channel + (channel.endsWith("/") ? "" : "/") + "api/v7.0/cdk/forward/sync/solr"; + String retval = channel + (channel.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/sync/solr"; return Pair.of(retval, false); } } else { @@ -233,7 +272,7 @@ private static Document deleteRootPid(SolrAccess solrAccess, String pid) throws private static String query(SolrAccess sAccess, String pid) throws IOException { String query = String.format("root.pid:\"%s\"",pid); if (query.startsWith("root.pid:\"uuid")) { - int threshold = KConfiguration.getInstance().getConfiguration().getInt("cdk.reharvest.items.threshold",80000); + int threshold = KConfiguration.getInstance().getConfiguration().getInt("cdk.reharvest.items.threshold",300); Document doc = sAccess.requestWithSelectReturningXml("q="+URLEncoder.encode( query, "UTF-8")); int numFound = numFound(doc); From 64635ab9744747429d01add124c72b85b3e3155c Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 21 Apr 2024 12:59:59 +0200 Subject: [PATCH 109/235] Debug messages --- .../main/java/cz/incad/kramerius/ReharvestPids.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java index 35d91d0be9..6e8eae7289 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java @@ -117,7 +117,10 @@ private static void reharvestPID(String pid, String onlyShowConfiguration) XMLUtils.print(deleteByQuery, writer); LOGGER.info("Delete by query "+writer.toString()); } else { - String s = SolrUtils.sendToDest(getDestinationUpdateUrl(), parallelProcess.getClient(), deleteByQuery); + StringWriter writer = new StringWriter(); + XMLUtils.print(deleteByQuery, writer); + LOGGER.info("Delete by query; only show query and number off docs :"+writer.toString()); + //String s = SolrUtils.sendToDest(getDestinationUpdateUrl(), parallelProcess.getClient(), deleteByQuery); } collections.remove(leader); @@ -192,6 +195,7 @@ private static void reharvestPIDFromGivenCollections(String pid, List co } else { ParallelProcessImpl reharvest = new ParallelProcessImpl(); String config = org.apache.commons.io.IOUtils.toString(new FileInputStream(harvestFile), "UTF-8"); + LOGGER.info(String.format("Configuration %s" ,config)); reharvest.migrate(harvestFile); } @@ -272,10 +276,11 @@ private static Document deleteRootPid(SolrAccess solrAccess, String pid) throws private static String query(SolrAccess sAccess, String pid) throws IOException { String query = String.format("root.pid:\"%s\"",pid); if (query.startsWith("root.pid:\"uuid")) { - int threshold = KConfiguration.getInstance().getConfiguration().getInt("cdk.reharvest.items.threshold",300); - + + int threshold = KConfiguration.getInstance().getConfiguration().getInt("cdk.reharvest.items.threshold",30000); Document doc = sAccess.requestWithSelectReturningXml("q="+URLEncoder.encode( query, "UTF-8")); int numFound = numFound(doc); + LOGGER.info(String.format( "Number of found documents %d", numFound)); if (numFound > threshold) { throw new IllegalStateException("Too many items to reharvest"); } From a719504896fad30cca7a40b50dd3bbedc0b669a6 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 21 Apr 2024 14:17:21 +0200 Subject: [PATCH 110/235] Version --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 05c0422e33..33e608f4b0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ #org.gradle.java.home=c:\\Users\\happy\\Programs\\jdk18 group=cz.incad.kramerius -version=7.0.23_dev3_cdk +version=7.0.33_dev9_cdk From f484e2a2271c8d5336602906826dd4755e221ddf Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 21 Apr 2024 14:27:19 +0200 Subject: [PATCH 111/235] Version --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 33e608f4b0..6faca7dd06 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ #org.gradle.java.home=c:\\Users\\happy\\Programs\\jdk18 group=cz.incad.kramerius -version=7.0.33_dev9_cdk +version=7.0.33_dev11_cdk From c35f18179a02c1ea71c0651205d23dd238700266 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 21 Apr 2024 14:40:31 +0200 Subject: [PATCH 112/235] Add dependency --- gradle.properties | 2 +- search/build.gradle | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 6faca7dd06..839616cf9c 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ #org.gradle.java.home=c:\\Users\\happy\\Programs\\jdk18 group=cz.incad.kramerius -version=7.0.33_dev11_cdk +version=7.0.33_dev12_cdk diff --git a/search/build.gradle b/search/build.gradle index 410a81727a..a37f38b5de 100644 --- a/search/build.gradle +++ b/search/build.gradle @@ -50,6 +50,8 @@ dependencies { //api project(':processes:dnntflag') //api project(':processes:nkp-logs') + api project(':processes:cdkreharvest') + api 'javax.servlet:jstl:1.1.2' api 'taglibs:standard:1.1.2' From c9d3e57e55052a8bc42b2af150e6ed979d4abd8f Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 21 Apr 2024 21:29:21 +0200 Subject: [PATCH 113/235] Reharvest resurces --- .../incad/kramerius/reharvest_public_v5.xml | 86 ++++++++++++++++++ .../cz/incad/kramerius/reharvest_v5.xml | 87 +++++++++++++++++++ .../cz/incad/kramerius/reharvest_v7.xml | 74 ++++++++++++++++ 3 files changed, 247 insertions(+) create mode 100644 processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_public_v5.xml create mode 100644 processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_v5.xml create mode 100644 processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_v7.xml diff --git a/processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_public_v5.xml b/processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_public_v5.xml new file mode 100644 index 0000000000..7deec5e6e4 --- /dev/null +++ b/processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_public_v5.xml @@ -0,0 +1,86 @@ + + + + $iteration.dl$ + $iteration.dl$-reharvest + reharvest + + + + + $iteration.url$ + search + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + $iteration.fquery$ + + + + 1 + + + + + + + search + + 80 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + $iteration.dl$ + $iteration.dl$ + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + $iteration.dl$ + + + + + + + \ No newline at end of file diff --git a/processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_v5.xml b/processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_v5.xml new file mode 100644 index 0000000000..43198b329e --- /dev/null +++ b/processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_v5.xml @@ -0,0 +1,87 @@ + + + + $iteration.dl$ + $iteration.dl$-reharvest + reharvest + + + + + $iteration.url$ + select + + PID + 5000 + CURSOR + + $iteration.user$ + $iteration.pass$ + + $iteration.fquery$ + + + + + 1 + + + + + + + select + + 80 + + $iteration.url$ + + $iteration.user$ + $iteration.pass$ + + + true + root_pid + PID + + + $destination.url$ + + select + + + cdk.collection + + K7 + + + + + $destination.url$/update + + + + + + + + + + $iteration.dl$ + $iteration.dl$ + + + + + + + PID root_pid dnnt-labels contains-dnnt-labels dostupnost fedora.model + + $iteration.dl$ + + + + + + + \ No newline at end of file diff --git a/processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_v7.xml b/processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_v7.xml new file mode 100644 index 0000000000..ac23cd2887 --- /dev/null +++ b/processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_v7.xml @@ -0,0 +1,74 @@ + + + + $iteration.dl$ + $iteration.dl$-reharvest + reharvest + + ` + + + $iteration.url$ + select + pid + 4000 + CURSOR + + $iteration.fquery$ + + + + 1 + + + + + $iteration.url$ + select + 80 + + pid + + true + root.pid + pid + + $destination.url$ + select + + + cdk.collection + + indexer_version pid root.pid model created modified indexed keywords.* geographic_* genres.* publishers.* publication_places.* authors authors.* titles.* title.* root.* own_parent.* own_pid_path own_model_path rels_ext_index.sort foster_parents.pids in_collections in_collections.direct level pid_paths date.* date_range_* date.str part.* issue.* page.* id_* count_* coords.* languages.* physical_locations.* shelf_locators accessibility has_tiles ds.* collection.* mdt ddt donator text_ocr licenses contains_licenses licenses_of_ancestors + + + + + $destination.url$/update + + + + + + + + + + $iteration.dl$ + $iteration.dl$ + + + + + + + pid root.pid licenses contains_licenses licenses_of_ancestors titles.* collection.* in_collections in_collections.* title.* titles.* text_ocr + + + $iteration.dl$ + + + + + + From 711c4004d2a300a0c8519cb8510f5ff484a8c43a Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 21 Apr 2024 21:58:09 +0200 Subject: [PATCH 114/235] Reharvet pid - only show configuration --- .../src/main/java/cz/incad/kramerius/ReharvestPids.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java index 6e8eae7289..d8dbd07d13 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java @@ -120,7 +120,7 @@ private static void reharvestPID(String pid, String onlyShowConfiguration) StringWriter writer = new StringWriter(); XMLUtils.print(deleteByQuery, writer); LOGGER.info("Delete by query; only show query and number off docs :"+writer.toString()); - //String s = SolrUtils.sendToDest(getDestinationUpdateUrl(), parallelProcess.getClient(), deleteByQuery); + String s = SolrUtils.sendToDest(getDestinationUpdateUrl(), parallelProcess.getClient(), deleteByQuery); } collections.remove(leader); @@ -140,7 +140,7 @@ private static boolean isOnlyShowConfiguration(String onlyShowConfiguration) { if (onlyShowConfiguration != null && ("onlyshowconfiguration".equals(onlyShowConfiguration.toLowerCase()) || "true".equals(onlyShowConfiguration))) { return true; } - return true; + return false; } private static void reharvestPIDFromGivenCollections(String pid, List collections, String onlyShowConfiguration) { From 524e02d59aec18dd7648b98d941ee4021b0b42c9 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 28 Apr 2024 08:32:26 +0200 Subject: [PATCH 115/235] Info resource + configuration --- .../admin/v10/proxy/ConnectedInfoResource.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java index 11d6a6daf0..adbefa3616 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java @@ -37,6 +37,7 @@ import cz.incad.kramerius.timestamps.Timestamp; import cz.incad.kramerius.timestamps.TimestampStore; import cz.incad.kramerius.timestamps.impl.SolrTimestamp; +import cz.incad.kramerius.utils.conf.KConfiguration; @Path("/admin/v7.0/connected") public class ConnectedInfoResource { @@ -162,6 +163,22 @@ public Response timestamps(@PathParam("library") String library) { } } + @GET + @Path("{library}/config") + @Produces(MediaType.APPLICATION_JSON) + public Response config(@PathParam("library") String library) { + JSONObject config = new JSONObject(); + String baseurl = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".baseurl"); + String api = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".api"); + boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + library + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + library + ".licenses") : false; + String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".forwardurl"); + config.put("baseurl", baseurl); + config.put("api", api); + config.put("licenses", channelAccess); + config.put("forwardurl", channel); + return Response.ok(config.toString()).build(); + } + @PUT @Path("{library}/timestamp") @Produces(MediaType.APPLICATION_JSON) From d7b73cd45d0d7db3e749801d74d730c733d41fd2 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 29 Apr 2024 12:33:29 +0200 Subject: [PATCH 116/235] Reharvest pid + delete items --- .../services/KubernetesProcessImpl.java | 56 +---- .../services/ParallelProcessImpl.java | 2 - .../iterators/solr/SolrCursorIterator.java | 6 +- .../kubernetes/KubernetesEnvSupport.java | 87 ++++++++ .../incad/kramerius/services/TestLogger.java | 43 ---- .../cz/incad/kramerius/services/UzeiTest.java | 12 -- .../kramerius/KubernetesReharvestProcess.java | 203 ++++++++++++++++++ .../cz/incad/kramerius/ReharvestPids.java | 15 +- .../rest/api/guice/ApiServletModule.java | 8 +- .../admin/v10/reharvest/ReharvestItem.java | 136 ++++++++++++ .../admin/v10/reharvest/ReharvestManager.java | 18 ++ .../v10/reharvest/ReharvestResource.java | 120 +++++++++++ .../impl/MemoryReharvestManagerImpl.java | 55 +++++ 13 files changed, 639 insertions(+), 122 deletions(-) create mode 100644 processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/kubernetes/KubernetesEnvSupport.java delete mode 100644 processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/TestLogger.java delete mode 100644 processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/UzeiTest.java create mode 100644 processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java index 0363a23467..dfad1dad68 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/KubernetesProcessImpl.java @@ -1,6 +1,8 @@ package cz.incad.kramerius.services; import cz.incad.kramerius.service.MigrateSolrIndexException; +import cz.incad.kramerius.services.utils.kubernetes.KubernetesEnvSupport; + import org.antlr.stringtemplate.StringTemplate; import org.antlr.stringtemplate.StringTemplateGroup; import org.antlr.stringtemplate.language.DefaultTemplateLexer; @@ -15,7 +17,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.net.URL; -import java.util.HashMap; import java.util.Map; import java.util.TimeZone; import java.util.logging.Logger; @@ -25,20 +26,9 @@ */ public class KubernetesProcessImpl { - public static final String TMSP_URL ="TIMESTAMP_URL"; - public static final String ITERATION_URL = "ITERATION_URL"; - public static final String CHECK_URL = "CHECK_URL"; - public static final String DESTINATION_URL = "DESTINATION_URL"; - public static final String CONFIG_SOURCE = "CONFIG_SOURCE"; + public static final String ONLY_SHOW_CONFIGURATION = "ONLY_SHOW_CONFIGURATION"; - public static final String ITERATION_PREFIX = "ITERATION_"; - public static final String CHECK_PREFIX = "CHECK_"; - public static final String DEST_PREFIX = "DESTINATION_"; - - public static final String TMSP_PREFIX = "TIMESTAMP_"; - - public static final Logger LOGGER = Logger.getLogger(KubernetesProcessImpl.class.getName()); public static void main(String[] args) throws IOException, MigrateSolrIndexException, IllegalAccessException, InstantiationException, SAXException, ParserConfigurationException, NoSuchMethodException, ClassNotFoundException { @@ -46,8 +36,8 @@ public static void main(String[] args) throws IOException, MigrateSolrIndexExcep TimeZone.setDefault(TimeZone.getTimeZone("Europe/Prague")); Map env = System.getenv(); - if (env.containsKey(CONFIG_SOURCE) || args.length > 0) { - String configSource = env.containsKey(CONFIG_SOURCE) ? env.get(CONFIG_SOURCE) : args[0]; + if (env.containsKey(KubernetesEnvSupport.CONFIG_SOURCE) || args.length > 0) { + String configSource = env.containsKey(KubernetesEnvSupport.CONFIG_SOURCE) ? env.get(KubernetesEnvSupport.CONFIG_SOURCE) : args[0]; InputStream stream = KubernetesProcessImpl.class.getResourceAsStream(configSource); if (configSource.trim().startsWith("file:///")) { URL fileUrl = new URL(configSource); @@ -55,29 +45,10 @@ public static void main(String[] args) throws IOException, MigrateSolrIndexExcep } if (stream != null) { - Map iteration = new HashMap<>(); - prefixVariables(env, iteration, ITERATION_PREFIX); - if (env.containsKey(ITERATION_URL)) { - iteration.put("url", env.get(ITERATION_URL)); - } - - Map check = new HashMap<>(); - prefixVariables(env, check, CHECK_PREFIX); - if (env.containsKey(CHECK_URL)) { - check.put("url", env.get(CHECK_URL)); - } - - Map destination = new HashMap<>(); - prefixVariables(env, destination, DEST_PREFIX); - if (env.containsKey(DESTINATION_URL)) { - destination.put("url", env.get(DESTINATION_URL)); - } - - Map timestamps = new HashMap<>(); - prefixVariables(env, timestamps, TMSP_PREFIX); - if (env.containsKey(TMSP_PREFIX)) { - destination.put("url", env.get(DESTINATION_URL)); - } + Map iteration = KubernetesEnvSupport.iterationMap(env); + Map check = KubernetesEnvSupport.checkMap(env); + Map destination = KubernetesEnvSupport.destinationMap(env); + Map timestamps = KubernetesEnvSupport.timestampMap(env, destination); StringTemplate template = new StringTemplate( IOUtils.toString(stream, "UTF-8"), DefaultTemplateLexer.class); @@ -106,13 +77,4 @@ public static void main(String[] args) throws IOException, MigrateSolrIndexExcep LOGGER.severe("No configuration"); } } - - private static void prefixVariables(Map env, Map templateMap, String prefix) { - env.keySet().forEach(key-> { - if (key.startsWith(prefix)) { - String name = key.substring(prefix.length()).toLowerCase(); - templateMap.put(name, env.get(key)); - } - }); - } } diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java index d1613726e4..a9e9236105 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/ParallelProcessImpl.java @@ -144,8 +144,6 @@ public static long waitUntilStartWorkingTimeImpl(String startTime, String endTi endDateTimeLT = endDateTimeLT.plusDays(1); } } - - Duration duration = Duration.between(current, startDateTimeLT); if (!duration.isNegative()) { diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/solr/SolrCursorIterator.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/solr/SolrCursorIterator.java index 23a3959516..e3ea15cbd7 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/solr/SolrCursorIterator.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/solr/SolrCursorIterator.java @@ -28,7 +28,7 @@ public SolrCursorIterator(String address, String masterQuery, String filterQuery super(address, masterQuery, filterQuery, endpoint, id, sorting, rows, user, pass); } - static Element pidsCursorQuery( Client client, String url, String mq, String cursor, int rows, String fq, String endpoint, String identifierField, String sorting, String user, String pass) throws ParserConfigurationException, SAXException, IOException { + public static Element pidsCursorQuery( Client client, String url, String mq, String cursor, int rows, String fq, String endpoint, String identifierField, String sorting, String user, String pass) throws ParserConfigurationException, SAXException, IOException { String fullQuery = null; if (StringUtils.isAnyString(fq)) { fullQuery = "?q="+mq + (cursor!= null ? String.format("&rows=%d&cursorMark=%s", rows, cursor) : String.format("&rows=%d&cursorMark=*", rows))+"&sort=" + URLEncoder.encode(sorting, "UTF-8")+"&fl="+identifierField+"&fq=" + URLEncoder.encode(fq,"UTF-8"); @@ -41,7 +41,7 @@ static Element pidsCursorQuery( Client client, String url, String mq, String cur } - static String findCursorMark(Element elm) { + public static String findCursorMark(Element elm) { Element element = XMLUtils.findElement(elm, new XMLUtils.ElementsFilter() { @Override public boolean acceptElement(Element element) { @@ -53,7 +53,7 @@ public boolean acceptElement(Element element) { return element != null ? element.getTextContent() : null; } - static String findQueryCursorMark(Element elm) { + public static String findQueryCursorMark(Element elm) { Element queryParams = XMLUtils.findElement(elm, new XMLUtils.ElementsFilter() { @Override public boolean acceptElement(Element element) { diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/kubernetes/KubernetesEnvSupport.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/kubernetes/KubernetesEnvSupport.java new file mode 100644 index 0000000000..6f8c464349 --- /dev/null +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/kubernetes/KubernetesEnvSupport.java @@ -0,0 +1,87 @@ +package cz.incad.kramerius.services.utils.kubernetes; + +import java.util.HashMap; +import java.util.Map; + +public class KubernetesEnvSupport { + + public static final String TMSP_URL = "TIMESTAMP_URL"; + public static final String ITERATION_URL = "ITERATION_URL"; + public static final String CHECK_URL = "CHECK_URL"; + public static final String DESTINATION_URL = "DESTINATION_URL"; + public static final String CONFIG_SOURCE = "CONFIG_SOURCE"; + public static final String ITERATION_PREFIX = "ITERATION_"; + public static final String CHECK_PREFIX = "CHECK_"; + public static final String DEST_PREFIX = "DESTINATION_"; + + public static final String PROXY_API_URL = "PROXY_API_URL"; + public static final String REHARVEST_URL = "REHARVEST_URL"; + + + public static final String TMSP_PREFIX = "TIMESTAMP_"; + public static final String PROXY_API_PREFIX = "PROXY_API_"; + public static final String REHARVEST_PREFIX = "REHARVEST_"; + + public static void prefixVariables(Map env, Map templateMap, String prefix) { + env.keySet().forEach(key -> { + if (key.startsWith(prefix)) { + String name = key.substring(prefix.length()).toLowerCase(); + templateMap.put(name, env.get(key)); + } + }); + } + + public static Map timestampMap(Map env, Map destination) { + Map timestamps = new HashMap<>(); + prefixVariables(env, timestamps, TMSP_PREFIX); + if (env.containsKey(TMSP_PREFIX)) { + destination.put("url", env.get(DESTINATION_URL)); + } + return timestamps; + } + + public static Map destinationMap(Map env) { + Map destination = new HashMap<>(); + prefixVariables(env, destination, DEST_PREFIX); + if (env.containsKey(DESTINATION_URL)) { + destination.put("url", env.get(DESTINATION_URL)); + } + return destination; + } + + public static Map checkMap(Map env) { + Map check = new HashMap<>(); + prefixVariables(env, check, CHECK_PREFIX); + if (env.containsKey(CHECK_URL)) { + check.put("url", env.get(CHECK_URL)); + } + return check; + } + + public static Map iterationMap(Map env) { + Map iteration = new HashMap<>(); + prefixVariables(env, iteration, ITERATION_PREFIX); + if (env.containsKey(ITERATION_URL)) { + iteration.put("url", env.get(ITERATION_URL)); + } + return iteration; + } + + public static Map proxyMap(Map env) { + Map proxy = new HashMap<>(); + prefixVariables(env, proxy, PROXY_API_PREFIX); + if (env.containsKey(PROXY_API_URL)) { + proxy.put("url", env.get(PROXY_API_URL)); + } + return proxy; + } + public static Map reharvestMap(Map env) { + Map proxy = new HashMap<>(); + prefixVariables(env, proxy, REHARVEST_PREFIX); + if (env.containsKey(REHARVEST_URL)) { + proxy.put("url", env.get(REHARVEST_URL)); + } + return proxy; + } + +} diff --git a/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/TestLogger.java b/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/TestLogger.java deleted file mode 100644 index 85a21be13c..0000000000 --- a/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/TestLogger.java +++ /dev/null @@ -1,43 +0,0 @@ -package cz.incad.kramerius.services; - -import org.junit.Test; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -public class TestLogger { - - - public static class Osoba { - private String jmeno; - private String prijmeni; - - - public Osoba(String jmeno, String prijmeni) { - this.jmeno = jmeno; - this.prijmeni = prijmeni; - } - - public String getPrijmeni() { - return prijmeni; - } - - public String getJmeno() { - return jmeno; - } - - @Override - public String toString() { - return "Osoba{" + - "jmeno='" + jmeno + '\'' + - ", prijmeni='" + prijmeni + '\'' + - '}'; - } - } - - -} - diff --git a/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/UzeiTest.java b/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/UzeiTest.java deleted file mode 100644 index 272525f880..0000000000 --- a/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/UzeiTest.java +++ /dev/null @@ -1,12 +0,0 @@ -package cz.incad.kramerius.services; - -import java.util.UUID; - -public class UzeiTest { - - public static void main(String[] args) { - UUID randomUUID = UUID.randomUUID(); - - System.out.println(randomUUID); - } -} diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java new file mode 100644 index 0000000000..fdc96ef6fc --- /dev/null +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -0,0 +1,203 @@ +package cz.incad.kramerius; + +import static cz.incad.kramerius.services.iterators.utils.IterationUtils.pidsToIterationItem; +import static cz.incad.kramerius.services.utils.SolrUtils.findAllPids; + +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.io.UnsupportedEncodingException; +import java.net.URL; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.ws.rs.core.MediaType; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; + +import org.antlr.stringtemplate.StringTemplate; +import org.antlr.stringtemplate.language.DefaultTemplateLexer; +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.IOUtils; +import org.json.JSONArray; +import org.json.JSONObject; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.UniformInterfaceException; +import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.config.ClientConfig; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.api.json.JSONConfiguration; + +import cz.incad.kramerius.services.KubernetesProcessImpl; +import cz.incad.kramerius.services.ParallelProcessImpl; +import cz.incad.kramerius.services.utils.kubernetes.KubernetesEnvSupport; +import cz.incad.kramerius.utils.XMLUtils; + +import static cz.incad.kramerius.services.iterators.solr.SolrCursorIterator.*; + + +public class KubernetesReharvestProcess { + + public static final Logger LOGGER = Logger.getLogger(KubernetesReharvestProcess.class.getName()); + + + protected static Client buildClient() { + //Client client = Client.create(); + ClientConfig cc = new DefaultClientConfig(); + cc.getProperties().put( + ClientConfig.PROPERTY_FOLLOW_REDIRECTS, true); + cc.getFeatures().put(JSONConfiguration.FEATURE_POJO_MAPPING, Boolean.TRUE); + return Client.create(cc); + } + + + public static void main(String[] args) throws UnsupportedEncodingException, ParserConfigurationException { + + TimeZone.setDefault(TimeZone.getTimeZone("Europe/Prague")); + + Map env = System.getenv(); + Map iterationMap = KubernetesEnvSupport.iterationMap(env); + Map proxyMap = KubernetesEnvSupport.reharvestMap(env); + if (proxyMap.containsKey("url")) { + Client client= buildClient(); + String wurl = proxyMap.get("url"); + if (!wurl.endsWith("/")) { + wurl = wurl +"/"; + } + WebResource r = client.resource(wurl + "top"); + + ClientResponse clientResponse = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + if (clientResponse.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + + String t = clientResponse.getEntity(String.class); + + JSONObject itemObject = new JSONObject(t); + String id = itemObject.getString("id"); + + LinkedHashSet allPids = new LinkedHashSet<>(); + JSONArray pids = itemObject.getJSONArray("pids"); + for (int i = 0; i < pids.length(); i++) { + String pid = pids.getString(i); + allPids.add(pid); + String iterationUrl = iterationMap.get("url"); + String masterQuery = "*:*"; + String filterQuery = "root.pid:\""+pid+"\""; + try { + String cursorMark = null; + String queryCursorMark = null; + do { + Element element = pidsCursorQuery(client, iterationUrl, masterQuery, cursorMark, 100, filterQuery, "select", "compositeId+pid", "compositeId asc", "", ""); + cursorMark = findCursorMark(element); + queryCursorMark = findQueryCursorMark(element); + allPids.addAll(findAllPids(element)); + } while((cursorMark != null && queryCursorMark != null) && !cursorMark.equals(queryCursorMark)); + } catch (ParserConfigurationException | SAXException | IOException e) { + throw new RuntimeException(e); + } + } + + //System.out.println("Number of pids "+allPids.size()); + List allPidsList = new ArrayList<>(); + allPidsList.addAll(allPids); + allPidsList.sort(String::compareTo); + + int batchSize = 42; + int batches = allPidsList.size() / batchSize; + if (allPidsList.size() % batchSize > 0) { + batches = batches + 1; + } + + for (int i = 0; i < batches; i++) { + int min = i* batchSize; + int max = Math.min((i+1)*batchSize, allPidsList.size()); + + Document deleteBatch = XMLUtils.crateDocument("delete"); + List batchPids = allPidsList.subList(min, max); + + batchPids.forEach(pid-> { + Element idElm = deleteBatch.createElement("id"); + idElm.setTextContent(pid.trim()); + deleteBatch.getDocumentElement().appendChild(idElm); + }) ; + + try { + StringWriter writer = new StringWriter(); + XMLUtils.print(deleteBatch, writer); + System.out.println(writer.toString()); + } catch (TransformerException e) { + e.printStackTrace(); + } + } + + } else if (clientResponse.getStatus() == ClientResponse.Status.NOT_FOUND.getStatusCode()) { + LOGGER.log(Level.SEVERE,"No item for reharvest"); + } else { + throw new UniformInterfaceException(clientResponse); + } + + +// Map iteration = KubernetesEnvSupport.iterationMap(env); +// Map check = KubernetesEnvSupport.checkMap(env); +// Map destination = KubernetesEnvSupport.destinationMap(env); +// Map timestamps = KubernetesEnvSupport.timestampMap(env, destination); +// +// StringTemplate template = new StringTemplate( +// IOUtils.toString(stream, "UTF-8"), DefaultTemplateLexer.class); + + } + + + // +// if (env.containsKey(KubernetesEnvSupport.CONFIG_SOURCE) || args.length > 0) { +// String configSource = env.containsKey(KubernetesEnvSupport.CONFIG_SOURCE) ? env.get(KubernetesEnvSupport.CONFIG_SOURCE) : args[0]; +// InputStream stream = KubernetesProcessImpl.class.getResourceAsStream(configSource); +// if (configSource.trim().startsWith("file:///")) { +// URL fileUrl = new URL(configSource); +// stream = fileUrl.openStream(); +// } +// if (stream != null) { +// +// Map iteration = KubernetesEnvSupport.iterationMap(env); +// Map check = KubernetesEnvSupport.checkMap(env); +// Map destination = KubernetesEnvSupport.destinationMap(env); +// Map timestamps = KubernetesEnvSupport.timestampMap(env, destination); +// +// StringTemplate template = new StringTemplate( +// IOUtils.toString(stream, "UTF-8"), DefaultTemplateLexer.class); +// +// template.setAttribute("iteration", iteration); +// template.setAttribute("check", check); +// template.setAttribute("destination", destination); +// template.setAttribute("timestamp", timestamps); +// +// String configuration = template.toString(); +// LOGGER.info("Loading configuration "+configuration); +// +// File tmpFile = File.createTempFile("temp", "file"); +// FileUtils.write(tmpFile, configuration, "UTF-8"); +// +// +// if (!env.containsKey(ONLY_SHOW_CONFIGURATION)) { +// ParallelProcessImpl migr = new ParallelProcessImpl(); +// migr.migrate(tmpFile); +// } +// +// } else { +// LOGGER.severe(String.format("Cannot find resource %s", configSource)); +// } +// } + } + +} diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java index d8dbd07d13..a51ca1fc8d 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java @@ -63,17 +63,6 @@ public class ReharvestPids { public static final Logger LOGGER = Logger.getLogger(ReharvestPids.class.getName()); - /** - * args[0] - action (ADD/REMOVE), from lp.st process/parameters - * args[1] - authToken - * args[2] - target (pid:uuid:123, or pidlist:uuid:123;uuid:345;uuid:789, or pidlist_file:/home/kramerius/.kramerius/import-dnnt/grafiky.txt - * In case of pidlist pids must be separated with ';'. Convenient separator ',' won't work due to way how params are stored in database and transferred to process. - *

- * args[3] - licence ('dnnt', 'dnnto', 'public_domain', etc.) - * @throws TransformerException - * @throws ParserConfigurationException - * @throws MigrateSolrIndexException - */ public static void main(String[] args) throws IOException, SolrServerException, RepositoryException, ResourceIndexException, TransformerException, ParserConfigurationException, MigrateSolrIndexException { if (args.length < 2) { throw new RuntimeException("Not enough arguments."); @@ -125,7 +114,6 @@ private static void reharvestPID(String pid, String onlyShowConfiguration) collections.remove(leader); collections.add(0, leader); - reharvestPIDFromGivenCollections(pid, collections, onlyShowConfiguration); } else { DefaultPropertiesInstances props = new DefaultPropertiesInstances(); @@ -153,9 +141,7 @@ private static void reharvestPIDFromGivenCollections(String pid, List co continue; } - Pair iterationUrl = iterationUrl(ac); - Map iteration = new HashMap<>(); iteration.put("url", iterationUrl.getKey()); iteration.put("dl", ac); @@ -193,6 +179,7 @@ private static void reharvestPIDFromGivenCollections(String pid, List co String config = org.apache.commons.io.IOUtils.toString(new FileInputStream(harvestFile), "UTF-8"); LOGGER.info(String.format("Configuration %s" ,config)); } else { + // safra ?? ParallelProcessImpl reharvest = new ParallelProcessImpl(); String config = org.apache.commons.io.IOUtils.toString(new FileInputStream(harvestFile), "UTF-8"); LOGGER.info(String.format("Configuration %s" ,config)); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java index e904a7b60e..99903f8045 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java @@ -32,6 +32,9 @@ //import cz.incad.kramerius.keycloak.KeycloakProxy; import cz.incad.kramerius.rest.apiNew.admin.v10.license.LicensesResource; import cz.incad.kramerius.rest.apiNew.admin.v10.proxy.ConnectedInfoResource; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestResource; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.impl.MemoryReharvestManagerImpl; import cz.incad.kramerius.rest.apiNew.admin.v10.rights.RightsResource; import cz.incad.kramerius.rest.api.k5.admin.statistics.StatisticsResource; import cz.incad.kramerius.rest.apiNew.admin.v10.rights.RolesResource; @@ -146,9 +149,12 @@ protected void configureServlets() { // cdk bind(TimestampStore.class).to(SolrTimestampStore.class).asEagerSingleton(); bind(Instances.class).to(DefaultPropertiesInstances.class).asEagerSingleton(); + bind(ReharvestManager.class).to(MemoryReharvestManagerImpl.class).asEagerSingleton(); bind(ProxyFilter.class).to(DefaultFilter.class); - bind(ConnectedInfoResource.class); + bind(ConnectedInfoResource.class); + //bind(ReharvestResource.class); + bind(ReharvestResource.class); // api Map parameters = new HashMap(); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java new file mode 100644 index 0000000000..b54a55b724 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java @@ -0,0 +1,136 @@ +package cz.incad.kramerius.rest.apiNew.admin.v10.reharvest; + +import java.text.ParseException; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.TimeZone; + +import org.apache.commons.lang3.time.FastDateFormat; +import org.json.JSONArray; +import org.json.JSONObject; + +import cz.incad.kramerius.utils.StringUtils; + +public class ReharvestItem { + +// static enum ReharvestItemState { +// open, closed; +// } + + //public static FastDateFormat FORMAT = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.sss'Z'", TimeZone.getTimeZone("UTC")); + + + private String id; + private String name; + private String state; + private String type; + private List pids= new ArrayList<>(); + private Instant timestamp = Instant.now(); + + + public ReharvestItem(String id, String name, String state, List pids) { + super(); + this.id = id; + this.name = name; + this.state = state; + this.pids = pids; + } + + public ReharvestItem(String id) { + super(); + this.id = id; + } + + + public String getId() { + return id; + } + + public String getName() { + return name; + } + public void setName(String name) { + this.name = name; + } + + public String getState() { + return state; + } + + public void setState(String state) { + this.state = state; + } + + public List getPids() { + return pids; + } + + public void setPids(List pids) { + this.pids = pids; + } + + public Instant getTimestamp() { + return timestamp; + } + + public void setTimestamp(Instant timestamp) { + this.timestamp = timestamp; + } + + public String getType() { + return type; + } + + public void setType(String type) { + this.type = type; + } + + public JSONObject toJSON() { + JSONObject obj = new JSONObject(); + obj.put("id", this.id); + obj.put("name", this.name); + if (this.state != null) { + obj.put("state", this.state); + } + if (this.type != null) { + obj.put("type", this.type); + } + + JSONArray jsonArray = new JSONArray(); + this.pids.forEach(jsonArray::put); + obj.put("pids", jsonArray); + + obj.put("timestamp",DateTimeFormatter.ISO_INSTANT.format(this.timestamp)); + + return obj; + } + + public static ReharvestItem fromJSON(JSONObject json) throws ParseException { + String id = json.getString("id"); + String name= json.getString("name"); + JSONArray array= json.getJSONArray("pids"); + List pids = new ArrayList<>(); + for (int i = 0; i < array.length(); i++) { + pids.add(array.getString(i)); + } + + String state = json.optString("state"); + String type = json.optString("type"); + ReharvestItem item = new ReharvestItem(id); + item.setName(name); + item.setPids(pids); + + item.setState(StringUtils.isAnyString(state) ? state : "open"); + item.setType(StringUtils.isAnyString(type) ? type : "root.pid"); + + if (json.has("timestamp")) { + String timestamp = json.getString("timestamp"); + item.setTimestamp( Instant.from(DateTimeFormatter.ISO_INSTANT.parse(timestamp))); + } + + return item; + } +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java new file mode 100644 index 0000000000..828369ae08 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java @@ -0,0 +1,18 @@ +package cz.incad.kramerius.rest.apiNew.admin.v10.reharvest; + +import java.util.List; + +public interface ReharvestManager { + + public void register(ReharvestItem item); + + public List getItems(); + + public ReharvestItem getTopItem(); + + + public ReharvestItem getItemById(String id); + + + public void deregister(String id); +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java new file mode 100644 index 0000000000..5674253f25 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -0,0 +1,120 @@ +package cz.incad.kramerius.rest.apiNew.admin.v10.reharvest; + +import java.text.ParseException; +import java.time.Instant; +import java.util.UUID; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.ws.rs.Consumes; +import javax.ws.rs.DELETE; +import javax.ws.rs.GET; +import javax.ws.rs.PUT; +import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; + +import com.google.inject.Inject; + +@Path("/admin/v7.0/reharvest") +public class ReharvestResource { + + public Logger LOGGER = Logger.getLogger(ReharvestResource.class.getName()); + + @Inject + private ReharvestManager reharvestManager; + + @GET + @Produces(MediaType.APPLICATION_JSON) + public Response getHarvests() { + JSONArray jsonArray = new JSONArray(); + this.reharvestManager.getItems().forEach(ri -> { + jsonArray.put(ri.toJSON()); + }); + return Response.ok(jsonArray.toString()).build(); + } + + @GET + @Path("{id}") + @Produces(MediaType.APPLICATION_JSON) + public Response getHarvest(@PathParam("id") String id) { + ReharvestItem item = this.reharvestManager.getItemById(id); + if (item != null) { + return Response.ok(item.toJSON().toString()).build(); + } else { + return Response.status(Response.Status.NOT_FOUND).build(); + } + } + + @DELETE + @Path("{id}") + @Produces(MediaType.APPLICATION_JSON) + public Response delete(@PathParam("id") String id) { + ReharvestItem item = this.reharvestManager.getItemById(id); + if (item != null) { + this.reharvestManager.deregister(id); + return Response.ok(item.toJSON().toString()).build(); + } else { + return Response.status(Response.Status.NOT_FOUND).build(); + } + } + + @PUT + @Produces(MediaType.APPLICATION_JSON) + @Consumes(MediaType.APPLICATION_JSON) + public Response register(String json) { + try { + JSONObject jsonObj = new JSONObject(json); + if (!jsonObj.has("id")) { + jsonObj.put("id", UUID.randomUUID().toString()); + } + ReharvestItem item = ReharvestItem.fromJSON(jsonObj); + if (item != null) { + item.setTimestamp(Instant.now()); + this.reharvestManager.register(item); + return Response.ok(item.toJSON().toString()).build(); + } else { + return Response.status(Response.Status.NOT_FOUND).build(); + } + } catch (JSONException | ParseException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); + } + } + + + + @GET + @Path("top") + @Produces(MediaType.APPLICATION_JSON) + public Response getTopHarvest() { + ReharvestItem topItem = this.reharvestManager.getTopItem(); + if (topItem != null) { + return Response.ok(topItem.toJSON().toString()).build(); + } else { + return Response.status(Response.Status.NOT_FOUND).build(); + } + } + + // +// +// @PUT +// @Path("{id}/state") +// @Produces(MediaType.APPLICATION_JSON) +// public Response getTopHarvest(@PathParam("id") String id, @QueryParam("status") String status) { +// ReharvestItem itemById = reharvestManager.getItemById(id); +// if (itemById != null) { +// itemById.setState(status); +// return Response.ok(itemById.toJSON().toString()).build(); +// } else { +// return Response.status(Response.Status.NOT_FOUND).build(); +// } +// } + +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java new file mode 100644 index 0000000000..b2e8a19648 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java @@ -0,0 +1,55 @@ +package cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.impl; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; + + +public class MemoryReharvestManagerImpl implements ReharvestManager { + + private List items = new ArrayList<>(); + private Map mapper = new HashMap<>(); + + public MemoryReharvestManagerImpl() { + super(); + } + + public void register(ReharvestItem item) { + this.items.add(item); + sortItems(); + this.mapper.put(item.getId(), item); + } + + public List getItems() { + sortItems(); + return this.items; + } + + private void sortItems() { + this.items.sort((item1, item2)-> { + return item1.getTimestamp().compareTo(item2.getTimestamp()); + }); + } + + public ReharvestItem getTopItem() { + sortItems(); + return this.items.size() > 0 ? this.items.get(0) : null; + } + + public ReharvestItem getItemById(String id) { + return this.mapper.get(id); + } + + public void deregister(String id) { + ReharvestItem ritem = this.mapper.get(id); + if (ritem != null) { + this.mapper.remove(id); + this.items.remove(ritem); + } + } + +} From 057a1fac39eef371afd9e044c13042b9ac3b296b Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 29 Apr 2024 18:30:18 +0200 Subject: [PATCH 117/235] New version --- gradle.properties | 2 +- .../ParallelProcessImplTest_REPLICATE.java | 2022 +++++++++-------- .../kramerius/KubernetesReharvestProcess.java | 233 +- .../cz/incad/kramerius/ReharvestPids.java | 359 --- .../incad/kramerius/utils/ReharvestUtils.java | 233 ++ .../admin/v10/reharvest/ReharvestManager.java | 2 +- .../v10/reharvest/ReharvestResource.java | 33 +- .../impl/MemoryReharvestManagerImpl.java | 6 +- 8 files changed, 1355 insertions(+), 1535 deletions(-) delete mode 100644 processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java create mode 100644 processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java diff --git a/gradle.properties b/gradle.properties index 839616cf9c..813b689860 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ #org.gradle.java.home=c:\\Users\\happy\\Programs\\jdk18 group=cz.incad.kramerius -version=7.0.33_dev12_cdk +version=7.0.33_dev17_cdk diff --git a/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/ParallelProcessImplTest_REPLICATE.java b/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/ParallelProcessImplTest_REPLICATE.java index f35c0a4636..7045389023 100644 --- a/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/ParallelProcessImplTest_REPLICATE.java +++ b/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/ParallelProcessImplTest_REPLICATE.java @@ -50,7 +50,8 @@ public void testFullProcess_EXISTS() throws IOException, IllegalAccessException, .createMock(); // --- 20 updated --- - String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A00000834-c06c-49ed-91e9-d54a0f8c8571%22+OR+%22uuid%3A00001fe4-1d09-42e2-845a-0201dd900e87%22+OR+%22uuid%3A0000211b-7c66-4730-b343-8ea2ef427f99%22+OR+%22uuid%3A000026fa-3695-4af4-aaaa-50557d8c2c6d%22+OR+%22uuid%3A0000329c-2940-89be-717b-82cac72948a8%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%401%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%402%22+OR+%22uuid%3A00003962-b232-4055-9a77-8d8db9487b90%22+OR+%22uuid%3A00003bb1-429d-11e2-849c-005056a60003%22+OR+%22uuid%3A000040ee-eda5-49d3-ae14-510a74921cda%22+OR+%22uuid%3A000047d7-3b67-4f08-a7cf-e843db917097%22+OR+%22uuid%3A000059eb-d782-4285-857a-41b370dc1afd%22+OR+%22uuid%3A00006174-e3da-488b-9a18-eef7b3d5e527%22+OR+%22uuid%3A00006bd9-100d-4c89-97e6-3529e99c45be%22+OR+%22uuid%3A00006e78-04c4-4fa0-9a9d-d7ba78f7c341%22+OR+%22uuid%3A00007ca2-054e-4fde-b73c-81f55a282ebc%22+OR+%22uuid%3A00007e8c-3f3a-4f73-ab70-0f95b1024f59%22+OR+%22uuid%3A000091fb-dc07-4594-9300-8a503dc09210%22+OR+%22uuid%3A0000a0c8-41b2-4fdd-9597-619b3a7ff965%22)&fl=PID+collection&wt=xml&rows=20"; + //http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3A00000834-c06c-49ed-91e9-d54a0f8c8571%22+OR+%22uuid%3A00001fe4-1d09-42e2-845a-0201dd900e87%22+OR+%22uuid%3A0000211b-7c66-4730-b343-8ea2ef427f99%22+OR+%22uuid%3A000026fa-3695-4af4-aaaa-50557d8c2c6d%22+OR+%22uuid%3A0000329c-2940-89be-717b-82cac72948a8%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%401%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%402%22+OR+%22uuid%3A00003962-b232-4055-9a77-8d8db9487b90%22+OR+%22uuid%3A00003bb1-429d-11e2-849c-005056a60003%22+OR+%22uuid%3A000040ee-eda5-49d3-ae14-510a74921cda%22+OR+%22uuid%3A000047d7-3b67-4f08-a7cf-e843db917097%22+OR+%22uuid%3A000059eb-d782-4285-857a-41b370dc1afd%22+OR+%22uuid%3A00006174-e3da-488b-9a18-eef7b3d5e527%22+OR+%22uuid%3A00006bd9-100d-4c89-97e6-3529e99c45be%22+OR+%22uuid%3A00006e78-04c4-4fa0-9a9d-d7ba78f7c341%22+OR+%22uuid%3A00007ca2-054e-4fde-b73c-81f55a282ebc%22+OR+%22uuid%3A00007e8c-3f3a-4f73-ab70-0f95b1024f59%22+OR+%22uuid%3A000091fb-dc07-4594-9300-8a503dc09210%22+OR+%22uuid%3A0000a0c8-41b2-4fdd-9597-619b3a7ff965%22)&fl=pid+collection+cdk.leader+cdk.collection+cdk.licenses+cdk.licenses_of_ancestors+cdk.contains_licenses++root.pid+compositeId&wt=xml&rows=20 + String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3A00000834-c06c-49ed-91e9-d54a0f8c8571%22+OR+%22uuid%3A00001fe4-1d09-42e2-845a-0201dd900e87%22+OR+%22uuid%3A0000211b-7c66-4730-b343-8ea2ef427f99%22+OR+%22uuid%3A000026fa-3695-4af4-aaaa-50557d8c2c6d%22+OR+%22uuid%3A0000329c-2940-89be-717b-82cac72948a8%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%401%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%402%22+OR+%22uuid%3A00003962-b232-4055-9a77-8d8db9487b90%22+OR+%22uuid%3A00003bb1-429d-11e2-849c-005056a60003%22+OR+%22uuid%3A000040ee-eda5-49d3-ae14-510a74921cda%22+OR+%22uuid%3A000047d7-3b67-4f08-a7cf-e843db917097%22+OR+%22uuid%3A000059eb-d782-4285-857a-41b370dc1afd%22+OR+%22uuid%3A00006174-e3da-488b-9a18-eef7b3d5e527%22+OR+%22uuid%3A00006bd9-100d-4c89-97e6-3529e99c45be%22+OR+%22uuid%3A00006e78-04c4-4fa0-9a9d-d7ba78f7c341%22+OR+%22uuid%3A00007ca2-054e-4fde-b73c-81f55a282ebc%22+OR+%22uuid%3A00007e8c-3f3a-4f73-ab70-0f95b1024f59%22+OR+%22uuid%3A000091fb-dc07-4594-9300-8a503dc09210%22+OR+%22uuid%3A0000a0c8-41b2-4fdd-9597-619b3a7ff965%22)&fl=pid+collection+cdk.leader+cdk.collection+cdk.licenses+cdk.licenses_of_ancestors+cdk.contains_licenses++root.pid+compositeId&wt=xml&rows=20"; String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("exists/1.xml"), "UTF-8"); List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); // ---------- @@ -94,7 +95,8 @@ public void testFullProcess_EXISTS() throws IOException, IllegalAccessException, Capture firstArg = newCapture(); EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - + + Capture secondArg = newCapture(); EasyMock.expect(updateResponseBuilder.entity(capture(secondArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); @@ -147,6 +149,10 @@ public void testFullProcess_EXISTS() throws IOException, IllegalAccessException, // start whole process process.migrate(configurationFile); + + + System.out.println(firstArg); + assertThat(Input.fromReader(new StringReader((String) firstArg.getValue()))) .and(Input.fromStream(this.getClass().getResourceAsStream("exists/batches/1_batch.xml"))) @@ -180,1012 +186,1012 @@ public void testFullProcess_EXISTS() throws IOException, IllegalAccessException, } - /** Simple copy index - data are copied */ - @Test - public void testFullProcess_NOT_EXISTS() throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { - InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("config.replicate.mlp.xml"); - String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); - - InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("logfile_log"); - String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); - File logFile = File.createTempFile("junit","logfile"); - FileUtils.write(logFile, logFileContent, "UTF-8"); - - String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); - File configurationFile = File.createTempFile("junit","conf"); - FileUtils.write(configurationFile, configurationContent, "UTF-8"); - - - Client client = EasyMock.createMock(Client.class); - ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class) - .addMockedMethod("buildClient") - .createMock(); - - // --- 20 indexed --- - String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A00000834-c06c-49ed-91e9-d54a0f8c8571%22+OR+%22uuid%3A00001fe4-1d09-42e2-845a-0201dd900e87%22+OR+%22uuid%3A0000211b-7c66-4730-b343-8ea2ef427f99%22+OR+%22uuid%3A000026fa-3695-4af4-aaaa-50557d8c2c6d%22+OR+%22uuid%3A0000329c-2940-89be-717b-82cac72948a8%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%401%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%402%22+OR+%22uuid%3A00003962-b232-4055-9a77-8d8db9487b90%22+OR+%22uuid%3A00003bb1-429d-11e2-849c-005056a60003%22+OR+%22uuid%3A000040ee-eda5-49d3-ae14-510a74921cda%22+OR+%22uuid%3A000047d7-3b67-4f08-a7cf-e843db917097%22+OR+%22uuid%3A000059eb-d782-4285-857a-41b370dc1afd%22+OR+%22uuid%3A00006174-e3da-488b-9a18-eef7b3d5e527%22+OR+%22uuid%3A00006bd9-100d-4c89-97e6-3529e99c45be%22+OR+%22uuid%3A00006e78-04c4-4fa0-9a9d-d7ba78f7c341%22+OR+%22uuid%3A00007ca2-054e-4fde-b73c-81f55a282ebc%22+OR+%22uuid%3A00007e8c-3f3a-4f73-ab70-0f95b1024f59%22+OR+%22uuid%3A000091fb-dc07-4594-9300-8a503dc09210%22+OR+%22uuid%3A0000a0c8-41b2-4fdd-9597-619b3a7ff965%22)&fl=PID+collection&wt=xml&rows=20"; - String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/1.xml"), "UTF-8"); - List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); - - String secondReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A00000834-c06c-49ed-91e9-d54a0f8c8571%22+OR+%22uuid%3A00001fe4-1d09-42e2-845a-0201dd900e87%22+OR+%22uuid%3A0000211b-7c66-4730-b343-8ea2ef427f99%22+OR+%22uuid%3A000026fa-3695-4af4-aaaa-50557d8c2c6d%22+OR+%22uuid%3A0000329c-2940-89be-717b-82cac72948a8%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%401%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%402%22+OR+%22uuid%3A00003962-b232-4055-9a77-8d8db9487b90%22+OR+%22uuid%3A00003bb1-429d-11e2-849c-005056a60003%22+OR+%22uuid%3A000040ee-eda5-49d3-ae14-510a74921cda%22+OR+%22uuid%3A000047d7-3b67-4f08-a7cf-e843db917097%22+OR+%22uuid%3A000059eb-d782-4285-857a-41b370dc1afd%22+OR+%22uuid%3A00006174-e3da-488b-9a18-eef7b3d5e527%22+OR+%22uuid%3A00006bd9-100d-4c89-97e6-3529e99c45be%22+OR+%22uuid%3A00006e78-04c4-4fa0-9a9d-d7ba78f7c341%22+OR+%22uuid%3A00007ca2-054e-4fde-b73c-81f55a282ebc%22+OR+%22uuid%3A00007e8c-3f3a-4f73-ab70-0f95b1024f59%22+OR+%22uuid%3A000091fb-dc07-4594-9300-8a503dc09210%22+OR+%22uuid%3A0000a0c8-41b2-4fdd-9597-619b3a7ff965%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; - String secondRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/1_fetch.xml"), "UTF-8"); - List mocksFromSecondFetchCall = webCallExpect(client, secondReqFetch, secondRespFetch); - // ---------- - - // --- 40 indexed --- - String thirdReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A0000a140-892d-48e3-851c-c0722578db50%22+OR+%22uuid%3A0000a210-40ad-48c6-a324-b4bd5fefe0f1%22+OR+%22uuid%3A0000a64f-d935-4260-acce-88833c115963%22+OR+%22uuid%3A0000ab19-c07d-41ac-9c6c-b03240271247%22+OR+%22uuid%3A0000abeb-c9ef-4ee9-8db1-536132bfac84%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%401%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%402%22+OR+%22uuid%3A0000b509-ed4f-447e-895f-d17d0d4ad0a3%22+OR+%22uuid%3A0000bbf9-0881-4e13-a793-457ed22bc58d%22+OR+%22uuid%3A0000c101-821b-4e21-95a3-13110119e10c%22+OR+%22uuid%3A0000c509-14e4-40b6-b555-f9ad46bce4c0%22+OR+%22uuid%3A0000c71a-aec6-4772-95f7-9412fd19f74c%22+OR+%22uuid%3A0000d5b8-44be-4473-b5f3-9e1f2437ce61%22+OR+%22uuid%3A0000dc41-0c83-4fe3-b013-081b57c6db8e%22+OR+%22uuid%3A0000dcd8-9a40-4b91-b3be-c7ffa05f6295%22+OR+%22uuid%3A0000dedc-cf17-11e1-acea-005056a60003%22+OR+%22uuid%3A0000e1cf-0621-4f8a-9bbc-cfdf4d25f096%22+OR+%22uuid%3A0000ed1b-31a8-4275-96fe-e6c80ef3e1b1%22+OR+%22uuid%3A0000ffd8-d89c-42db-a818-537144d80261%22)&fl=PID+collection&wt=xml&rows=20"; - String thirdResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/2.xml"), "UTF-8"); - List mocksFromThirdCall = webCallExpect(client, thirdReq, thirdResp); - - String thirdReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A0000a140-892d-48e3-851c-c0722578db50%22+OR+%22uuid%3A0000a210-40ad-48c6-a324-b4bd5fefe0f1%22+OR+%22uuid%3A0000a64f-d935-4260-acce-88833c115963%22+OR+%22uuid%3A0000ab19-c07d-41ac-9c6c-b03240271247%22+OR+%22uuid%3A0000abeb-c9ef-4ee9-8db1-536132bfac84%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%401%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%402%22+OR+%22uuid%3A0000b509-ed4f-447e-895f-d17d0d4ad0a3%22+OR+%22uuid%3A0000bbf9-0881-4e13-a793-457ed22bc58d%22+OR+%22uuid%3A0000c101-821b-4e21-95a3-13110119e10c%22+OR+%22uuid%3A0000c509-14e4-40b6-b555-f9ad46bce4c0%22+OR+%22uuid%3A0000c71a-aec6-4772-95f7-9412fd19f74c%22+OR+%22uuid%3A0000d5b8-44be-4473-b5f3-9e1f2437ce61%22+OR+%22uuid%3A0000dc41-0c83-4fe3-b013-081b57c6db8e%22+OR+%22uuid%3A0000dcd8-9a40-4b91-b3be-c7ffa05f6295%22+OR+%22uuid%3A0000dedc-cf17-11e1-acea-005056a60003%22+OR+%22uuid%3A0000e1cf-0621-4f8a-9bbc-cfdf4d25f096%22+OR+%22uuid%3A0000ed1b-31a8-4275-96fe-e6c80ef3e1b1%22+OR+%22uuid%3A0000ffd8-d89c-42db-a818-537144d80261%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; - String thirdRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/2_fetch.xml"), "UTF-8"); - List mocksFromThirdFetchCall = webCallExpect(client, thirdReqFetch, thirdRespFetch); - // ---------- - - // --- 60 indexed --- - String fourthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A00010fd5-d7c8-46fa-b87a-d87ca2aa84a5%22+OR+%22uuid%3A00011d48-9699-4b21-b000-9b0c34995b63%22+OR+%22uuid%3A00012082-f119-4370-9c0c-b6055137d6d8%22+OR+%22uuid%3A00012289-fae2-44a5-ba57-db7e5ab1159b%22+OR+%22uuid%3A0001257f-b598-4a0e-9c1f-8b199c115929%22+OR+%22uuid%3A00012835-2127-42b5-b598-6ece58276164%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%2F%401%22+OR+%22uuid%3A00013431-3c37-40db-9b4c-e87bbdd1d8eb%22+OR+%22uuid%3A00013618-6699-4721-b8d1-61b3a96bd969%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%2F%401%22+OR+%22uuid%3A00013afa-b718-42b1-b9bf-7f633b97c747%22+OR+%22uuid%3A00013d61-2317-43d2-9a69-39686e37c527%22+OR+%22uuid%3A00015940-ee76-415d-8169-797789406cff%22+OR+%22uuid%3A00016234-018a-4cce-90a4-5eaecd78122f%22+OR+%22uuid%3A00016cdf-2591-465d-bfd0-fece6ae08bf1%22+OR+%22uuid%3A0001714e-bbb4-4d7a-856a-c79aa618bfa2%22+OR+%22uuid%3A00017177-b182-46da-b88a-1377215389ea%22+OR+%22uuid%3A0001759e-cad0-467b-b61a-cfe1381a76f6%22)&fl=PID+collection&wt=xml&rows=20"; - String fourthResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/3.xml"), "UTF-8"); - List mocksFromFourthCall = webCallExpect(client, fourthReq, fourthResp); - - String fourthReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A00010fd5-d7c8-46fa-b87a-d87ca2aa84a5%22+OR+%22uuid%3A00011d48-9699-4b21-b000-9b0c34995b63%22+OR+%22uuid%3A00012082-f119-4370-9c0c-b6055137d6d8%22+OR+%22uuid%3A00012289-fae2-44a5-ba57-db7e5ab1159b%22+OR+%22uuid%3A0001257f-b598-4a0e-9c1f-8b199c115929%22+OR+%22uuid%3A00012835-2127-42b5-b598-6ece58276164%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%2F%401%22+OR+%22uuid%3A00013431-3c37-40db-9b4c-e87bbdd1d8eb%22+OR+%22uuid%3A00013618-6699-4721-b8d1-61b3a96bd969%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%2F%401%22+OR+%22uuid%3A00013afa-b718-42b1-b9bf-7f633b97c747%22+OR+%22uuid%3A00013d61-2317-43d2-9a69-39686e37c527%22+OR+%22uuid%3A00015940-ee76-415d-8169-797789406cff%22+OR+%22uuid%3A00016234-018a-4cce-90a4-5eaecd78122f%22+OR+%22uuid%3A00016cdf-2591-465d-bfd0-fece6ae08bf1%22+OR+%22uuid%3A0001714e-bbb4-4d7a-856a-c79aa618bfa2%22+OR+%22uuid%3A00017177-b182-46da-b88a-1377215389ea%22+OR+%22uuid%3A0001759e-cad0-467b-b61a-cfe1381a76f6%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; - String fourthRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/3_fetch.xml"), "UTF-8"); - List mocksFromFourthFetchCall = webCallExpect(client, fourthReqFetch, fourthRespFetch); - // ---------- - - - // --- 80 indexed --- - String fifthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A000182c3-0215-4103-90ce-9f912a3caf02%22+OR+%22uuid%3A00018da8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001900d-e592-4ad4-8f61-21289fa0108c%22+OR+%22uuid%3A000199e3-6c01-4055-b620-d809be8c34c9%22+OR+%22uuid%3A00019d0e-ebcf-4aa0-ae46-e85403f867d0%22+OR+%22uuid%3A0001a28e-d40f-4caa-b99d-56b6b272be10%22+OR+%22uuid%3A0001b5ad-a2aa-43ea-912c-f472f7d0a786%22+OR+%22uuid%3A0001c4ba-bedf-4dbd-beb4-f87baf5bec3c%22+OR+%22uuid%3A0001c89d-0b86-448c-91c1-688f6882d548%22+OR+%22uuid%3A0001caa1-2fd5-4e0d-a74a-516286ee9414%22+OR+%22uuid%3A0001cd7d-446b-4696-b10d-a7115c6e4ed7%22+OR+%22uuid%3A0001cff1-9c7c-47ac-a2f0-340514a12852%22+OR+%22uuid%3A0001d02f-af3e-4d9d-9cae-31f1373aeffb%22+OR+%22uuid%3A0001dbc9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbca-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcb-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcc-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcd-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbce-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcf-3105-11e9-8847-005056a2b051%22)&fl=PID+collection&wt=xml&rows=20"; - String fifthResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/4.xml"), "UTF-8"); - List mocksFromFifthCall = webCallExpect(client, fifthReq, fifthResp); - - String fifthReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A000182c3-0215-4103-90ce-9f912a3caf02%22+OR+%22uuid%3A00018da8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001900d-e592-4ad4-8f61-21289fa0108c%22+OR+%22uuid%3A000199e3-6c01-4055-b620-d809be8c34c9%22+OR+%22uuid%3A00019d0e-ebcf-4aa0-ae46-e85403f867d0%22+OR+%22uuid%3A0001a28e-d40f-4caa-b99d-56b6b272be10%22+OR+%22uuid%3A0001b5ad-a2aa-43ea-912c-f472f7d0a786%22+OR+%22uuid%3A0001c4ba-bedf-4dbd-beb4-f87baf5bec3c%22+OR+%22uuid%3A0001c89d-0b86-448c-91c1-688f6882d548%22+OR+%22uuid%3A0001caa1-2fd5-4e0d-a74a-516286ee9414%22+OR+%22uuid%3A0001cd7d-446b-4696-b10d-a7115c6e4ed7%22+OR+%22uuid%3A0001cff1-9c7c-47ac-a2f0-340514a12852%22+OR+%22uuid%3A0001d02f-af3e-4d9d-9cae-31f1373aeffb%22+OR+%22uuid%3A0001dbc9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbca-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcb-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcc-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcd-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbce-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcf-3105-11e9-8847-005056a2b051%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; - String fifthRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/4_fetch.xml"), "UTF-8"); - List mocksFromFifthFetchCall = webCallExpect(client, fifthReqFetch, fifthRespFetch); - // ---------- - - - // --- 100 indexed --- - String sixthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A0001dbd0-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001de4b-75ee-4428-bb20-737cc2cee698%22+OR+%22uuid%3A0001e211-3e75-4c7e-871d-662c2bfc4471%22+OR+%22uuid%3A0001e71d-9c40-482c-b0b4-70fa1d62d4ed%22+OR+%22uuid%3A0001e8ae-02ab-49e7-bb84-d83291bc075e%22+OR+%22uuid%3A0001eff8-4618-417a-aede-4ab6b5274f8c%22+OR+%22uuid%3A0001f258-7472-42df-a994-c14817e8e9c5%22+OR+%22uuid%3A0001f761-5a0c-47cc-87de-db9513e4c305%22+OR+%22uuid%3A000202e1-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e2-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e3-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e4-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e5-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e6-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e7-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A00020759-510b-429d-a46c-ed0b25eb4a02%22+OR+%22uuid%3A000229fa-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000229fb-3105-11e9-8847-005056a2b051%22)&fl=PID+collection&wt=xml&rows=20"; - String sixthResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/5.xml"), "UTF-8"); - List mocksFromSixthCall = webCallExpect(client, sixthReq, sixthResp); - - String sixthReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A0001dbd0-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001de4b-75ee-4428-bb20-737cc2cee698%22+OR+%22uuid%3A0001e211-3e75-4c7e-871d-662c2bfc4471%22+OR+%22uuid%3A0001e71d-9c40-482c-b0b4-70fa1d62d4ed%22+OR+%22uuid%3A0001e8ae-02ab-49e7-bb84-d83291bc075e%22+OR+%22uuid%3A0001eff8-4618-417a-aede-4ab6b5274f8c%22+OR+%22uuid%3A0001f258-7472-42df-a994-c14817e8e9c5%22+OR+%22uuid%3A0001f761-5a0c-47cc-87de-db9513e4c305%22+OR+%22uuid%3A000202e1-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e2-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e3-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e4-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e5-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e6-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e7-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A00020759-510b-429d-a46c-ed0b25eb4a02%22+OR+%22uuid%3A000229fa-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000229fb-3105-11e9-8847-005056a2b051%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; - String sixthRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/5_fetch.xml"), "UTF-8"); - List mocksFromSixthFetchCall = webCallExpect(client, sixthReqFetch, sixthRespFetch); - // ---------- - - EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); - - // all updates in one check - WebResource updateResource = EasyMock.createMock(WebResource.class); - ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); - WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")).andReturn(updateResource).anyTimes(); - EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); - EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); - - //EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); - //Capture captureSingleArgument = EasyMock.newCa - Capture firstArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture secondArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(secondArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture thirdArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(thirdArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture fourthArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(fourthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture fifthArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(fifthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - - EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); - EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); - EasyMock.expect(updateResponse.getEntityInputStream()).andDelegateTo(new MockClientResponse()).anyTimes(); - - // final commit - WebResource commitResource = EasyMock.createMock(WebResource.class); - WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); - - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")).andReturn(commitResource).anyTimes(); - EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); - - EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, - commitResource, - commitResourceBuilder); - - // check & fetch - mocksFromSecondCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromSecondFetchCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - // check & fetch - mocksFromThirdCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromThirdFetchCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - // check & fetch - mocksFromFourthCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromFourthFetchCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - // check & fetch - mocksFromFifthCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromFifthFetchCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - - // check & fetch - mocksFromSixthCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromSixthFetchCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - process.setClient(client); - - // start whole process - process.migrate(configurationFile); - - - - assertThat(Input.fromReader(new StringReader((String) firstArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/1_batch.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - assertThat(Input.fromReader(new StringReader((String) secondArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/2_batch.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - assertThat(Input.fromReader(new StringReader((String) thirdArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/3_batch.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - assertThat(Input.fromReader(new StringReader((String) fourthArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/4_batch.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - assertThat(Input.fromReader(new StringReader((String) fifthArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/5_batch.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - - } - - - /** Simple copy index - data are updated but in solr cloud */ - @Test - public void testFullProcess_EXISTS_SOLR_CLOUD() throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { - InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("config.replicate.mlp_solrcloud.xml"); - String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); - - InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("logfile_log"); - String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); - File logFile = File.createTempFile("junit","logfile"); - FileUtils.write(logFile, logFileContent, "UTF-8"); - - String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); - File configurationFile = File.createTempFile("junit","conf"); - FileUtils.write(configurationFile, configurationContent, "UTF-8"); - - - Client client = EasyMock.createMock(Client.class); - ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class) - .addMockedMethod("buildClient") - .createMock(); - - // --- 20 updated --- - String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A00000834-c06c-49ed-91e9-d54a0f8c8571%22+OR+%22uuid%3A00001fe4-1d09-42e2-845a-0201dd900e87%22+OR+%22uuid%3A0000211b-7c66-4730-b343-8ea2ef427f99%22+OR+%22uuid%3A000026fa-3695-4af4-aaaa-50557d8c2c6d%22+OR+%22uuid%3A0000329c-2940-89be-717b-82cac72948a8%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%401%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%402%22+OR+%22uuid%3A00003962-b232-4055-9a77-8d8db9487b90%22+OR+%22uuid%3A00003bb1-429d-11e2-849c-005056a60003%22+OR+%22uuid%3A000040ee-eda5-49d3-ae14-510a74921cda%22+OR+%22uuid%3A000047d7-3b67-4f08-a7cf-e843db917097%22+OR+%22uuid%3A000059eb-d782-4285-857a-41b370dc1afd%22+OR+%22uuid%3A00006174-e3da-488b-9a18-eef7b3d5e527%22+OR+%22uuid%3A00006bd9-100d-4c89-97e6-3529e99c45be%22+OR+%22uuid%3A00006e78-04c4-4fa0-9a9d-d7ba78f7c341%22+OR+%22uuid%3A00007ca2-054e-4fde-b73c-81f55a282ebc%22+OR+%22uuid%3A00007e8c-3f3a-4f73-ab70-0f95b1024f59%22+OR+%22uuid%3A000091fb-dc07-4594-9300-8a503dc09210%22+OR+%22uuid%3A0000a0c8-41b2-4fdd-9597-619b3a7ff965%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; - String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("exists/1_solrcloud.xml"), "UTF-8"); - List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); - // ---------- - - // --- 40 updated --- - String thirdReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A0000a140-892d-48e3-851c-c0722578db50%22+OR+%22uuid%3A0000a210-40ad-48c6-a324-b4bd5fefe0f1%22+OR+%22uuid%3A0000a64f-d935-4260-acce-88833c115963%22+OR+%22uuid%3A0000ab19-c07d-41ac-9c6c-b03240271247%22+OR+%22uuid%3A0000abeb-c9ef-4ee9-8db1-536132bfac84%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%401%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%402%22+OR+%22uuid%3A0000b509-ed4f-447e-895f-d17d0d4ad0a3%22+OR+%22uuid%3A0000bbf9-0881-4e13-a793-457ed22bc58d%22+OR+%22uuid%3A0000c101-821b-4e21-95a3-13110119e10c%22+OR+%22uuid%3A0000c509-14e4-40b6-b555-f9ad46bce4c0%22+OR+%22uuid%3A0000c71a-aec6-4772-95f7-9412fd19f74c%22+OR+%22uuid%3A0000d5b8-44be-4473-b5f3-9e1f2437ce61%22+OR+%22uuid%3A0000dc41-0c83-4fe3-b013-081b57c6db8e%22+OR+%22uuid%3A0000dcd8-9a40-4b91-b3be-c7ffa05f6295%22+OR+%22uuid%3A0000dedc-cf17-11e1-acea-005056a60003%22+OR+%22uuid%3A0000e1cf-0621-4f8a-9bbc-cfdf4d25f096%22+OR+%22uuid%3A0000ed1b-31a8-4275-96fe-e6c80ef3e1b1%22+OR+%22uuid%3A0000ffd8-d89c-42db-a818-537144d80261%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; - String thirdResp = IOUtils.toString(this.getClass().getResourceAsStream("exists/2_solrcloud.xml"), "UTF-8"); - List mocksFromThirdCall = webCallExpect(client, thirdReq, thirdResp); - // ---------- - - // --- 60 updated --- - String fourthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A00010fd5-d7c8-46fa-b87a-d87ca2aa84a5%22+OR+%22uuid%3A00011d48-9699-4b21-b000-9b0c34995b63%22+OR+%22uuid%3A00012082-f119-4370-9c0c-b6055137d6d8%22+OR+%22uuid%3A00012289-fae2-44a5-ba57-db7e5ab1159b%22+OR+%22uuid%3A0001257f-b598-4a0e-9c1f-8b199c115929%22+OR+%22uuid%3A00012835-2127-42b5-b598-6ece58276164%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%2F%401%22+OR+%22uuid%3A00013431-3c37-40db-9b4c-e87bbdd1d8eb%22+OR+%22uuid%3A00013618-6699-4721-b8d1-61b3a96bd969%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%2F%401%22+OR+%22uuid%3A00013afa-b718-42b1-b9bf-7f633b97c747%22+OR+%22uuid%3A00013d61-2317-43d2-9a69-39686e37c527%22+OR+%22uuid%3A00015940-ee76-415d-8169-797789406cff%22+OR+%22uuid%3A00016234-018a-4cce-90a4-5eaecd78122f%22+OR+%22uuid%3A00016cdf-2591-465d-bfd0-fece6ae08bf1%22+OR+%22uuid%3A0001714e-bbb4-4d7a-856a-c79aa618bfa2%22+OR+%22uuid%3A00017177-b182-46da-b88a-1377215389ea%22+OR+%22uuid%3A0001759e-cad0-467b-b61a-cfe1381a76f6%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; - String fourthResp = IOUtils.toString(this.getClass().getResourceAsStream("exists/3_solrcloud.xml"), "UTF-8"); - List mocksFromFourthCall = webCallExpect(client, fourthReq, fourthResp); - // ---------- - - // --- 80 updated --- - String fifthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A000182c3-0215-4103-90ce-9f912a3caf02%22+OR+%22uuid%3A00018da8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001900d-e592-4ad4-8f61-21289fa0108c%22+OR+%22uuid%3A000199e3-6c01-4055-b620-d809be8c34c9%22+OR+%22uuid%3A00019d0e-ebcf-4aa0-ae46-e85403f867d0%22+OR+%22uuid%3A0001a28e-d40f-4caa-b99d-56b6b272be10%22+OR+%22uuid%3A0001b5ad-a2aa-43ea-912c-f472f7d0a786%22+OR+%22uuid%3A0001c4ba-bedf-4dbd-beb4-f87baf5bec3c%22+OR+%22uuid%3A0001c89d-0b86-448c-91c1-688f6882d548%22+OR+%22uuid%3A0001caa1-2fd5-4e0d-a74a-516286ee9414%22+OR+%22uuid%3A0001cd7d-446b-4696-b10d-a7115c6e4ed7%22+OR+%22uuid%3A0001cff1-9c7c-47ac-a2f0-340514a12852%22+OR+%22uuid%3A0001d02f-af3e-4d9d-9cae-31f1373aeffb%22+OR+%22uuid%3A0001dbc9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbca-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcb-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcc-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcd-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbce-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcf-3105-11e9-8847-005056a2b051%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; - String fifthResp = IOUtils.toString(this.getClass().getResourceAsStream("exists/4_solrcloud.xml"), "UTF-8"); - List mocksFromFifthCall = webCallExpect(client, fifthReq, fifthResp); - // ---------- - - // --- 100 updated --- - String sixthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A0001dbd0-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001de4b-75ee-4428-bb20-737cc2cee698%22+OR+%22uuid%3A0001e211-3e75-4c7e-871d-662c2bfc4471%22+OR+%22uuid%3A0001e71d-9c40-482c-b0b4-70fa1d62d4ed%22+OR+%22uuid%3A0001e8ae-02ab-49e7-bb84-d83291bc075e%22+OR+%22uuid%3A0001eff8-4618-417a-aede-4ab6b5274f8c%22+OR+%22uuid%3A0001f258-7472-42df-a994-c14817e8e9c5%22+OR+%22uuid%3A0001f761-5a0c-47cc-87de-db9513e4c305%22+OR+%22uuid%3A000202e1-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e2-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e3-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e4-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e5-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e6-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e7-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A00020759-510b-429d-a46c-ed0b25eb4a02%22+OR+%22uuid%3A000229fa-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000229fb-3105-11e9-8847-005056a2b051%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; - String sixthResp = IOUtils.toString(this.getClass().getResourceAsStream("exists/5_solrcloud.xml"), "UTF-8"); - List mocksFromSixthCall = webCallExpect(client, sixthReq, sixthResp); - // ---------- - - - EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); - - // all updates in one check; (anyString) - WebResource updateResource = EasyMock.createMock(WebResource.class); - ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); - WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")).andReturn(updateResource).anyTimes(); - EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); - EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); - - - Capture firstArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture secondArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(secondArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture thirdArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(thirdArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture fourthArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(fourthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture fifthArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(fifthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - - //EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); - - EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); - EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); - EasyMock.expect(updateResponse.getEntityInputStream()).andDelegateTo(new MockClientResponse()).anyTimes(); - - // final commit - - WebResource commitResource = EasyMock.createMock(WebResource.class); - WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); - - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")).andReturn(commitResource).anyTimes(); - EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); - - EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, - commitResource, - commitResourceBuilder); - mocksFromSecondCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromThirdCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromFourthCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromFifthCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromSixthCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - process.setClient(client); - - // start whole process - process.migrate(configurationFile); - - assertThat(Input.fromReader(new StringReader((String) firstArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("exists/batches/1_batch_solrcloud.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - assertThat(Input.fromReader(new StringReader((String) secondArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("exists/batches/2_batch_solrcloud.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - assertThat(Input.fromReader(new StringReader((String) thirdArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("exists/batches/3_batch_solrcloud.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - assertThat(Input.fromReader(new StringReader((String) fourthArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("exists/batches/4_batch_solrcloud.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - assertThat(Input.fromReader(new StringReader((String) fifthArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("exists/batches/5_batch_solrcloud.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - } - - - - /** Simple copy index - data are indexed byt in solr cloud */ - @Test - public void testFullProcess_NOT_EXISTS_SOLR_CLOUD() throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { - InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("config.replicate.mlp_solrcloud.xml"); - String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); - - InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("logfile_log"); - String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); - File logFile = File.createTempFile("junit","logfile"); - FileUtils.write(logFile, logFileContent, "UTF-8"); - - String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); - File configurationFile = File.createTempFile("junit","conf"); - FileUtils.write(configurationFile, configurationContent, "UTF-8"); - - Client client = EasyMock.createMock(Client.class); - ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class) - .addMockedMethod("buildClient") - .createMock(); - - // --- 20 indexed --- - String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A00000834-c06c-49ed-91e9-d54a0f8c8571%22+OR+%22uuid%3A00001fe4-1d09-42e2-845a-0201dd900e87%22+OR+%22uuid%3A0000211b-7c66-4730-b343-8ea2ef427f99%22+OR+%22uuid%3A000026fa-3695-4af4-aaaa-50557d8c2c6d%22+OR+%22uuid%3A0000329c-2940-89be-717b-82cac72948a8%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%401%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%402%22+OR+%22uuid%3A00003962-b232-4055-9a77-8d8db9487b90%22+OR+%22uuid%3A00003bb1-429d-11e2-849c-005056a60003%22+OR+%22uuid%3A000040ee-eda5-49d3-ae14-510a74921cda%22+OR+%22uuid%3A000047d7-3b67-4f08-a7cf-e843db917097%22+OR+%22uuid%3A000059eb-d782-4285-857a-41b370dc1afd%22+OR+%22uuid%3A00006174-e3da-488b-9a18-eef7b3d5e527%22+OR+%22uuid%3A00006bd9-100d-4c89-97e6-3529e99c45be%22+OR+%22uuid%3A00006e78-04c4-4fa0-9a9d-d7ba78f7c341%22+OR+%22uuid%3A00007ca2-054e-4fde-b73c-81f55a282ebc%22+OR+%22uuid%3A00007e8c-3f3a-4f73-ab70-0f95b1024f59%22+OR+%22uuid%3A000091fb-dc07-4594-9300-8a503dc09210%22+OR+%22uuid%3A0000a0c8-41b2-4fdd-9597-619b3a7ff965%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; - String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/1.xml"), "UTF-8"); - List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); - - String secondReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A00000834-c06c-49ed-91e9-d54a0f8c8571%22+OR+%22uuid%3A00001fe4-1d09-42e2-845a-0201dd900e87%22+OR+%22uuid%3A0000211b-7c66-4730-b343-8ea2ef427f99%22+OR+%22uuid%3A000026fa-3695-4af4-aaaa-50557d8c2c6d%22+OR+%22uuid%3A0000329c-2940-89be-717b-82cac72948a8%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%401%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%402%22+OR+%22uuid%3A00003962-b232-4055-9a77-8d8db9487b90%22+OR+%22uuid%3A00003bb1-429d-11e2-849c-005056a60003%22+OR+%22uuid%3A000040ee-eda5-49d3-ae14-510a74921cda%22+OR+%22uuid%3A000047d7-3b67-4f08-a7cf-e843db917097%22+OR+%22uuid%3A000059eb-d782-4285-857a-41b370dc1afd%22+OR+%22uuid%3A00006174-e3da-488b-9a18-eef7b3d5e527%22+OR+%22uuid%3A00006bd9-100d-4c89-97e6-3529e99c45be%22+OR+%22uuid%3A00006e78-04c4-4fa0-9a9d-d7ba78f7c341%22+OR+%22uuid%3A00007ca2-054e-4fde-b73c-81f55a282ebc%22+OR+%22uuid%3A00007e8c-3f3a-4f73-ab70-0f95b1024f59%22+OR+%22uuid%3A000091fb-dc07-4594-9300-8a503dc09210%22+OR+%22uuid%3A0000a0c8-41b2-4fdd-9597-619b3a7ff965%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; - String secondRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/1_fetch.xml"), "UTF-8"); - List mocksFromSecondFetchCall = webCallExpect(client, secondReqFetch, secondRespFetch); - // ---------- - - // --- 40 indexed --- - String thirdReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A0000a140-892d-48e3-851c-c0722578db50%22+OR+%22uuid%3A0000a210-40ad-48c6-a324-b4bd5fefe0f1%22+OR+%22uuid%3A0000a64f-d935-4260-acce-88833c115963%22+OR+%22uuid%3A0000ab19-c07d-41ac-9c6c-b03240271247%22+OR+%22uuid%3A0000abeb-c9ef-4ee9-8db1-536132bfac84%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%401%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%402%22+OR+%22uuid%3A0000b509-ed4f-447e-895f-d17d0d4ad0a3%22+OR+%22uuid%3A0000bbf9-0881-4e13-a793-457ed22bc58d%22+OR+%22uuid%3A0000c101-821b-4e21-95a3-13110119e10c%22+OR+%22uuid%3A0000c509-14e4-40b6-b555-f9ad46bce4c0%22+OR+%22uuid%3A0000c71a-aec6-4772-95f7-9412fd19f74c%22+OR+%22uuid%3A0000d5b8-44be-4473-b5f3-9e1f2437ce61%22+OR+%22uuid%3A0000dc41-0c83-4fe3-b013-081b57c6db8e%22+OR+%22uuid%3A0000dcd8-9a40-4b91-b3be-c7ffa05f6295%22+OR+%22uuid%3A0000dedc-cf17-11e1-acea-005056a60003%22+OR+%22uuid%3A0000e1cf-0621-4f8a-9bbc-cfdf4d25f096%22+OR+%22uuid%3A0000ed1b-31a8-4275-96fe-e6c80ef3e1b1%22+OR+%22uuid%3A0000ffd8-d89c-42db-a818-537144d80261%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; - String thirdResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/2.xml"), "UTF-8"); - List mocksFromThirdCall = webCallExpect(client, thirdReq, thirdResp); - - String thirdReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A0000a140-892d-48e3-851c-c0722578db50%22+OR+%22uuid%3A0000a210-40ad-48c6-a324-b4bd5fefe0f1%22+OR+%22uuid%3A0000a64f-d935-4260-acce-88833c115963%22+OR+%22uuid%3A0000ab19-c07d-41ac-9c6c-b03240271247%22+OR+%22uuid%3A0000abeb-c9ef-4ee9-8db1-536132bfac84%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%401%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%402%22+OR+%22uuid%3A0000b509-ed4f-447e-895f-d17d0d4ad0a3%22+OR+%22uuid%3A0000bbf9-0881-4e13-a793-457ed22bc58d%22+OR+%22uuid%3A0000c101-821b-4e21-95a3-13110119e10c%22+OR+%22uuid%3A0000c509-14e4-40b6-b555-f9ad46bce4c0%22+OR+%22uuid%3A0000c71a-aec6-4772-95f7-9412fd19f74c%22+OR+%22uuid%3A0000d5b8-44be-4473-b5f3-9e1f2437ce61%22+OR+%22uuid%3A0000dc41-0c83-4fe3-b013-081b57c6db8e%22+OR+%22uuid%3A0000dcd8-9a40-4b91-b3be-c7ffa05f6295%22+OR+%22uuid%3A0000dedc-cf17-11e1-acea-005056a60003%22+OR+%22uuid%3A0000e1cf-0621-4f8a-9bbc-cfdf4d25f096%22+OR+%22uuid%3A0000ed1b-31a8-4275-96fe-e6c80ef3e1b1%22+OR+%22uuid%3A0000ffd8-d89c-42db-a818-537144d80261%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; - String thirdRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/2_fetch.xml"), "UTF-8"); - List mocksFromThirdFetchCall = webCallExpect(client, thirdReqFetch, thirdRespFetch); - // ---------- - - // --- 60 indexed --- - String fourthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A00010fd5-d7c8-46fa-b87a-d87ca2aa84a5%22+OR+%22uuid%3A00011d48-9699-4b21-b000-9b0c34995b63%22+OR+%22uuid%3A00012082-f119-4370-9c0c-b6055137d6d8%22+OR+%22uuid%3A00012289-fae2-44a5-ba57-db7e5ab1159b%22+OR+%22uuid%3A0001257f-b598-4a0e-9c1f-8b199c115929%22+OR+%22uuid%3A00012835-2127-42b5-b598-6ece58276164%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%2F%401%22+OR+%22uuid%3A00013431-3c37-40db-9b4c-e87bbdd1d8eb%22+OR+%22uuid%3A00013618-6699-4721-b8d1-61b3a96bd969%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%2F%401%22+OR+%22uuid%3A00013afa-b718-42b1-b9bf-7f633b97c747%22+OR+%22uuid%3A00013d61-2317-43d2-9a69-39686e37c527%22+OR+%22uuid%3A00015940-ee76-415d-8169-797789406cff%22+OR+%22uuid%3A00016234-018a-4cce-90a4-5eaecd78122f%22+OR+%22uuid%3A00016cdf-2591-465d-bfd0-fece6ae08bf1%22+OR+%22uuid%3A0001714e-bbb4-4d7a-856a-c79aa618bfa2%22+OR+%22uuid%3A00017177-b182-46da-b88a-1377215389ea%22+OR+%22uuid%3A0001759e-cad0-467b-b61a-cfe1381a76f6%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; - String fourthResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/3.xml"), "UTF-8"); - List mocksFromFourthCall = webCallExpect(client, fourthReq, fourthResp); - - String fourthReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A00010fd5-d7c8-46fa-b87a-d87ca2aa84a5%22+OR+%22uuid%3A00011d48-9699-4b21-b000-9b0c34995b63%22+OR+%22uuid%3A00012082-f119-4370-9c0c-b6055137d6d8%22+OR+%22uuid%3A00012289-fae2-44a5-ba57-db7e5ab1159b%22+OR+%22uuid%3A0001257f-b598-4a0e-9c1f-8b199c115929%22+OR+%22uuid%3A00012835-2127-42b5-b598-6ece58276164%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%2F%401%22+OR+%22uuid%3A00013431-3c37-40db-9b4c-e87bbdd1d8eb%22+OR+%22uuid%3A00013618-6699-4721-b8d1-61b3a96bd969%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%2F%401%22+OR+%22uuid%3A00013afa-b718-42b1-b9bf-7f633b97c747%22+OR+%22uuid%3A00013d61-2317-43d2-9a69-39686e37c527%22+OR+%22uuid%3A00015940-ee76-415d-8169-797789406cff%22+OR+%22uuid%3A00016234-018a-4cce-90a4-5eaecd78122f%22+OR+%22uuid%3A00016cdf-2591-465d-bfd0-fece6ae08bf1%22+OR+%22uuid%3A0001714e-bbb4-4d7a-856a-c79aa618bfa2%22+OR+%22uuid%3A00017177-b182-46da-b88a-1377215389ea%22+OR+%22uuid%3A0001759e-cad0-467b-b61a-cfe1381a76f6%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; - String fourthRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/3_fetch.xml"), "UTF-8"); - List mocksFromFourthFetchCall = webCallExpect(client, fourthReqFetch, fourthRespFetch); - // ---------- - - - // --- 80 indexed --- - String fifthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A000182c3-0215-4103-90ce-9f912a3caf02%22+OR+%22uuid%3A00018da8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001900d-e592-4ad4-8f61-21289fa0108c%22+OR+%22uuid%3A000199e3-6c01-4055-b620-d809be8c34c9%22+OR+%22uuid%3A00019d0e-ebcf-4aa0-ae46-e85403f867d0%22+OR+%22uuid%3A0001a28e-d40f-4caa-b99d-56b6b272be10%22+OR+%22uuid%3A0001b5ad-a2aa-43ea-912c-f472f7d0a786%22+OR+%22uuid%3A0001c4ba-bedf-4dbd-beb4-f87baf5bec3c%22+OR+%22uuid%3A0001c89d-0b86-448c-91c1-688f6882d548%22+OR+%22uuid%3A0001caa1-2fd5-4e0d-a74a-516286ee9414%22+OR+%22uuid%3A0001cd7d-446b-4696-b10d-a7115c6e4ed7%22+OR+%22uuid%3A0001cff1-9c7c-47ac-a2f0-340514a12852%22+OR+%22uuid%3A0001d02f-af3e-4d9d-9cae-31f1373aeffb%22+OR+%22uuid%3A0001dbc9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbca-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcb-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcc-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcd-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbce-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcf-3105-11e9-8847-005056a2b051%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; - String fifthResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/4.xml"), "UTF-8"); - List mocksFromFifthCall = webCallExpect(client, fifthReq, fifthResp); - - String fifthReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A000182c3-0215-4103-90ce-9f912a3caf02%22+OR+%22uuid%3A00018da8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001900d-e592-4ad4-8f61-21289fa0108c%22+OR+%22uuid%3A000199e3-6c01-4055-b620-d809be8c34c9%22+OR+%22uuid%3A00019d0e-ebcf-4aa0-ae46-e85403f867d0%22+OR+%22uuid%3A0001a28e-d40f-4caa-b99d-56b6b272be10%22+OR+%22uuid%3A0001b5ad-a2aa-43ea-912c-f472f7d0a786%22+OR+%22uuid%3A0001c4ba-bedf-4dbd-beb4-f87baf5bec3c%22+OR+%22uuid%3A0001c89d-0b86-448c-91c1-688f6882d548%22+OR+%22uuid%3A0001caa1-2fd5-4e0d-a74a-516286ee9414%22+OR+%22uuid%3A0001cd7d-446b-4696-b10d-a7115c6e4ed7%22+OR+%22uuid%3A0001cff1-9c7c-47ac-a2f0-340514a12852%22+OR+%22uuid%3A0001d02f-af3e-4d9d-9cae-31f1373aeffb%22+OR+%22uuid%3A0001dbc9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbca-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcb-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcc-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcd-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbce-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcf-3105-11e9-8847-005056a2b051%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; - String fifthRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/4_fetch.xml"), "UTF-8"); - List mocksFromFifthFetchCall = webCallExpect(client, fifthReqFetch, fifthRespFetch); - // ---------- - - - // --- 100 indexed --- - String sixthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A0001dbd0-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001de4b-75ee-4428-bb20-737cc2cee698%22+OR+%22uuid%3A0001e211-3e75-4c7e-871d-662c2bfc4471%22+OR+%22uuid%3A0001e71d-9c40-482c-b0b4-70fa1d62d4ed%22+OR+%22uuid%3A0001e8ae-02ab-49e7-bb84-d83291bc075e%22+OR+%22uuid%3A0001eff8-4618-417a-aede-4ab6b5274f8c%22+OR+%22uuid%3A0001f258-7472-42df-a994-c14817e8e9c5%22+OR+%22uuid%3A0001f761-5a0c-47cc-87de-db9513e4c305%22+OR+%22uuid%3A000202e1-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e2-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e3-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e4-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e5-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e6-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e7-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A00020759-510b-429d-a46c-ed0b25eb4a02%22+OR+%22uuid%3A000229fa-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000229fb-3105-11e9-8847-005056a2b051%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; - String sixthResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/5.xml"), "UTF-8"); - List mocksFromSixthCall = webCallExpect(client, sixthReq, sixthResp); - - String sixthReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A0001dbd0-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001de4b-75ee-4428-bb20-737cc2cee698%22+OR+%22uuid%3A0001e211-3e75-4c7e-871d-662c2bfc4471%22+OR+%22uuid%3A0001e71d-9c40-482c-b0b4-70fa1d62d4ed%22+OR+%22uuid%3A0001e8ae-02ab-49e7-bb84-d83291bc075e%22+OR+%22uuid%3A0001eff8-4618-417a-aede-4ab6b5274f8c%22+OR+%22uuid%3A0001f258-7472-42df-a994-c14817e8e9c5%22+OR+%22uuid%3A0001f761-5a0c-47cc-87de-db9513e4c305%22+OR+%22uuid%3A000202e1-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e2-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e3-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e4-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e5-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e6-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e7-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A00020759-510b-429d-a46c-ed0b25eb4a02%22+OR+%22uuid%3A000229fa-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000229fb-3105-11e9-8847-005056a2b051%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; - String sixthRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/5_fetch.xml"), "UTF-8"); - List mocksFromSixthFetchCall = webCallExpect(client, sixthReqFetch, sixthRespFetch); - // ---------- - - EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); - - // all updates in one check - WebResource updateResource = EasyMock.createMock(WebResource.class); - ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); - WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")).andReturn(updateResource).anyTimes(); - EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); - EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); - - Capture firstArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture secondArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(secondArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture thirdArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(thirdArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture fourthArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(fourthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture fifthArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(fifthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - //EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); - - EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); - EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); - EasyMock.expect(updateResponse.getEntityInputStream()).andDelegateTo(new MockClientResponse()).anyTimes(); - - // final commit - WebResource commitResource = EasyMock.createMock(WebResource.class); - WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); - - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")).andReturn(commitResource).anyTimes(); - EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); - - EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, - commitResource, - commitResourceBuilder); - - // check & fetch - mocksFromSecondCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromSecondFetchCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - // check & fetch - mocksFromThirdCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromThirdFetchCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - // check & fetch - mocksFromFourthCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromFourthFetchCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - // check & fetch - mocksFromFifthCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromFifthFetchCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - - // check & fetch - mocksFromSixthCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromSixthFetchCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - process.setClient(client); - - // start whole process - process.migrate(configurationFile); - - - assertThat(Input.fromReader(new StringReader((String) firstArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/1_batch_solrcloud.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - assertThat(Input.fromReader(new StringReader((String) secondArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/2_batch_solrcloud.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - assertThat(Input.fromReader(new StringReader((String) thirdArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/3_batch_solrcloud.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - assertThat(Input.fromReader(new StringReader((String) fourthArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/4_batch_solrcloud.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - assertThat(Input.fromReader(new StringReader((String) fifthArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/5_batch_solrcloud.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - - } - - /** Copy and transform index K5->K7 - data are updated */ - @Test - public void testFullProcess_EXISTS_K7_TRANSFORM() throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { - - InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("config.replicate.mlp_k7.xml"); - String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); - - InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("k7exists/logfile_log"); - String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); - File logFile = File.createTempFile("junit","logfile"); - FileUtils.write(logFile, logFileContent, "UTF-8"); - - String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); - File configurationFile = File.createTempFile("junit","conf"); - FileUtils.write(configurationFile, configurationContent, "UTF-8"); - - - Client client = EasyMock.createMock(Client.class); - ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class) - .addMockedMethod("buildClient") - .createMock(); - - // --- 20 indexed --- - String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3A03173900-662a-11e3-9ea2-5ef3fc9ae867%22+OR+%22uuid%3A03173e90-9c18-11e3-a880-5ef3fc9ae867%22+OR+%22uuid%3A03218150-2e9e-11eb-bd18-005056827e51%22+OR+%22uuid%3A03218050-35fe-11e4-8e0d-005056827e51%22+OR+%22uuid%3A03174110-5186-11e3-8c6a-005056825209%22+OR+%22uuid%3A03174020-f2fa-11e6-bf7e-005056825209%22+OR+%22uuid%3A03173f70-5dc0-11e9-a9b4-5ef3fc9bb22f%22+OR+%22uuid%3A031740b9-d472-4b35-8697-3efc830fd7fe%22+OR+%22uuid%3A031744d0-ea62-11e6-9964-005056825209%22+OR+%22uuid%3A03218800-bf0f-11e2-9592-5ef3fc9bb22f%22+OR+%22uuid%3A032185c1-ee27-11e2-9439-005056825209%22+OR+%22uuid%3A03218604-eb98-461c-8033-3dd1512169ef%22+OR+%22uuid%3A032187e2-f47b-47b5-a9ba-b0e00f12e8e9%22+OR+%22uuid%3A03174850-1edb-4aa1-acb1-4054fe2820f9%22+OR+%22uuid%3A03174a30-dac4-11e3-93a3-005056825209%22+OR+%22uuid%3A03174829-8301-413b-a982-2cfe1e6444bb%22+OR+%22uuid%3A03174c99-4e3b-464e-a65e-5d3d840f1693%22+OR+%22uuid%3A03218ea0-5522-11e8-ab92-5ef3fc9ae867%22+OR+%22uuid%3A03218c97-a41a-ffc3-fe18-4610acb2152a%22+OR+%22uuid%3A03218e20-b96e-11e5-b404-005056825209%22)&fl=pid+collection&wt=xml&rows=20"; - String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("k7exists/1.xml"), "UTF-8"); - List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); - - - EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); - - // all updates in one check - WebResource updateResource = EasyMock.createMock(WebResource.class); - ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); - WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")).andReturn(updateResource).anyTimes(); - EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); - EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); - - //EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); - //Capture captureSingleArgument = EasyMock.newCa - Capture firstArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture secondArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(secondArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture thirdArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(thirdArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture fourthArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(fourthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture fifthArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(fifthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - - EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); - EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); - EasyMock.expect(updateResponse.getEntityInputStream()).andDelegateTo(new MockClientResponse()).anyTimes(); - - // final commit - WebResource commitResource = EasyMock.createMock(WebResource.class); - WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); - - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")).andReturn(commitResource).anyTimes(); - EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); - - EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, - commitResource, - commitResourceBuilder); - - // check & fetch - mocksFromSecondCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - process.setClient(client); - - // start whole process - process.migrate(configurationFile); - - - assertThat(Input.fromReader(new StringReader((String) firstArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("k7exists/k7batches/1_batch.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - - } - - /** Copy and transform index K5->K7 - data are updated but in solr cloud*/ - @Test - public void testFullProcess_EXISTS_K7_TRANSFORM_SOLR_CLOUD() throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { - - InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("config.replicate.mlp_k7_solrcloud.xml"); - String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); - - InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("k7exists/logfile_log"); - String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); - File logFile = File.createTempFile("junit","logfile"); - FileUtils.write(logFile, logFileContent, "UTF-8"); - - String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); - File configurationFile = File.createTempFile("junit","conf"); - FileUtils.write(configurationFile, configurationContent, "UTF-8"); - - - Client client = EasyMock.createMock(Client.class); - ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class) - .addMockedMethod("buildClient") - .createMock(); - - // --- 20 indexed --- - String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3A03173900-662a-11e3-9ea2-5ef3fc9ae867%22+OR+%22uuid%3A03173e90-9c18-11e3-a880-5ef3fc9ae867%22+OR+%22uuid%3A03218150-2e9e-11eb-bd18-005056827e51%22+OR+%22uuid%3A03218050-35fe-11e4-8e0d-005056827e51%22+OR+%22uuid%3A03174110-5186-11e3-8c6a-005056825209%22+OR+%22uuid%3A03174020-f2fa-11e6-bf7e-005056825209%22+OR+%22uuid%3A03173f70-5dc0-11e9-a9b4-5ef3fc9bb22f%22+OR+%22uuid%3A031740b9-d472-4b35-8697-3efc830fd7fe%22+OR+%22uuid%3A031744d0-ea62-11e6-9964-005056825209%22+OR+%22uuid%3A03218800-bf0f-11e2-9592-5ef3fc9bb22f%22+OR+%22uuid%3A032185c1-ee27-11e2-9439-005056825209%22+OR+%22uuid%3A03218604-eb98-461c-8033-3dd1512169ef%22+OR+%22uuid%3A032187e2-f47b-47b5-a9ba-b0e00f12e8e9%22+OR+%22uuid%3A03174850-1edb-4aa1-acb1-4054fe2820f9%22+OR+%22uuid%3A03174a30-dac4-11e3-93a3-005056825209%22+OR+%22uuid%3A03174829-8301-413b-a982-2cfe1e6444bb%22+OR+%22uuid%3A03174c99-4e3b-464e-a65e-5d3d840f1693%22+OR+%22uuid%3A03218ea0-5522-11e8-ab92-5ef3fc9ae867%22+OR+%22uuid%3A03218c97-a41a-ffc3-fe18-4610acb2152a%22+OR+%22uuid%3A03218e20-b96e-11e5-b404-005056825209%22)&fl=pid+collection+root.pid&wt=xml&rows=20"; - String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("k7exists/1_solrcloud.xml"), "UTF-8"); - List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); - - - EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); - - // all updates in one check - WebResource updateResource = EasyMock.createMock(WebResource.class); - ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); - WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")).andReturn(updateResource).anyTimes(); - EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); - EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); - - //EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); - //Capture captureSingleArgument = EasyMock.newCa - Capture firstArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture secondArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(secondArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture thirdArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(thirdArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture fourthArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(fourthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture fifthArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(fifthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - - EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); - EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); - EasyMock.expect(updateResponse.getEntityInputStream()).andDelegateTo(new MockClientResponse()).anyTimes(); - - // final commit - WebResource commitResource = EasyMock.createMock(WebResource.class); - WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); - - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")).andReturn(commitResource).anyTimes(); - EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); - - EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, - commitResource, - commitResourceBuilder); - - // check & fetch - mocksFromSecondCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - process.setClient(client); - - // start whole process - process.migrate(configurationFile); - - - assertThat(Input.fromReader(new StringReader((String) firstArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("k7exists/k7batches/1_batch_solrcloud.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - } - - /** Copy and transform index K5->K7 - data are copied */ - @Test - public void testFullProcess_NOT_EXISTS_K7_TRANSFORM() throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { - - InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("config.replicate.mlp_k7.xml"); - String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); - - InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("k7notexists/logfile_log"); - String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); - File logFile = File.createTempFile("junit","logfile"); - FileUtils.write(logFile, logFileContent, "UTF-8"); - - String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); - File configurationFile = File.createTempFile("junit","conf"); - FileUtils.write(configurationFile, configurationContent, "UTF-8"); - - - Client client = EasyMock.createMock(Client.class); - ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class) - .addMockedMethod("buildClient") - .createMock(); - - // --- 20 indexed --- - String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3Ae15e3a10-7774-11e7-89ee-5ef3fc9ae867%22+OR+%22uuid%3A420f4830-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Ae665ffa0-d6ad-11e7-8294-005056827e51%22+OR+%22uuid%3A88459bc0-cf86-11e7-a351-005056825209%22+OR+%22uuid%3A74bddf50-cf85-11e7-b059-5ef3fc9ae867%22+OR+%22uuid%3Ab7d82771-d69f-11e7-8294-005056827e51%22+OR+%22uuid%3A9f3f7bb0-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Af7217151-cf8a-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3A30bcc500-cf8f-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3Ae32eb050-d466-11e7-a047-005056825209%22+OR+%22uuid%3Ac39f12a0-d44a-11e7-a536-5ef3fc9ae867%22+OR+%22uuid%3Ab6632460-7641-11e4-9605-005056825209%22+OR+%22uuid%3A1e062920-d6be-11e7-9268-5ef3fc9ae867%22+OR+%22uuid%3Ab6c1a8f0-a730-11e2-b6da-005056827e52%22+OR+%22uuid%3A7c5e1e70-cc80-11e3-aec3-005056827e52%22+OR+%22uuid%3A8afd1150-1b3e-11e3-b62e-005056825209%22+OR+%22uuid%3A72356250-92c3-11e9-b601-005056825209%22+OR+%22uuid%3Aca058c00-929e-11e9-bc42-5ef3fc9ae867%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22)&fl=pid+collection&wt=xml&rows=20"; - String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/1.xml"), "UTF-8"); - List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); - - String secondReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3Ae15e3a10-7774-11e7-89ee-5ef3fc9ae867%22+OR+%22uuid%3A420f4830-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Ae665ffa0-d6ad-11e7-8294-005056827e51%22+OR+%22uuid%3A88459bc0-cf86-11e7-a351-005056825209%22+OR+%22uuid%3A74bddf50-cf85-11e7-b059-5ef3fc9ae867%22+OR+%22uuid%3Ab7d82771-d69f-11e7-8294-005056827e51%22+OR+%22uuid%3A9f3f7bb0-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Af7217151-cf8a-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3A30bcc500-cf8f-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3Ae32eb050-d466-11e7-a047-005056825209%22+OR+%22uuid%3Ac39f12a0-d44a-11e7-a536-5ef3fc9ae867%22+OR+%22uuid%3Ab6632460-7641-11e4-9605-005056825209%22+OR+%22uuid%3A1e062920-d6be-11e7-9268-5ef3fc9ae867%22+OR+%22uuid%3Ab6c1a8f0-a730-11e2-b6da-005056827e52%22+OR+%22uuid%3A7c5e1e70-cc80-11e3-aec3-005056827e52%22+OR+%22uuid%3A8afd1150-1b3e-11e3-b62e-005056825209%22+OR+%22uuid%3A72356250-92c3-11e9-b601-005056825209%22+OR+%22uuid%3Aca058c00-929e-11e9-bc42-5ef3fc9ae867%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; - String secondRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/1_fetch.xml"), "UTF-8"); - List mocksFromSecondFetchCall = webCallExpect(client, secondReqFetch, secondRespFetch); - // ---------- - - // --- 3 indexed --- - String thirdReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3A3b624880-96b5-11e2-9a08-005056827e52%22+OR+%22uuid%3A87d2a060-a692-11e2-8b87-005056827e51%22+OR+%22uuid%3A653cfcf0-3fac-11e7-b56f-005056827e52%22)&fl=pid+collection&wt=xml&rows=3"; - String thirdResp = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/1.xml"), "UTF-8"); - List mocksFromThirdCall = webCallExpect(client, thirdReq, thirdResp); - - String thirdReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A3b624880-96b5-11e2-9a08-005056827e52%22+OR+%22uuid%3A87d2a060-a692-11e2-8b87-005056827e51%22+OR+%22uuid%3A653cfcf0-3fac-11e7-b56f-005056827e52%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=3"; - String thirdRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/2_fetch.xml"), "UTF-8"); - List mocksFromThirdFetchCall = webCallExpect(client, thirdReqFetch, thirdRespFetch); - // ---------- - - - EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); - - // all updates in one check - WebResource updateResource = EasyMock.createMock(WebResource.class); - ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); - WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")).andReturn(updateResource).anyTimes(); - EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); - EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); - - //EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); - //Capture captureSingleArgument = EasyMock.newCa - Capture firstArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture secondArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(secondArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture thirdArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(thirdArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture fourthArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(fourthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture fifthArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(fifthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - - EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); - EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); - EasyMock.expect(updateResponse.getEntityInputStream()).andDelegateTo(new MockClientResponse()).anyTimes(); - - // final commit - WebResource commitResource = EasyMock.createMock(WebResource.class); - WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); - - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")).andReturn(commitResource).anyTimes(); - EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); - - EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, - commitResource, - commitResourceBuilder); - - // check & fetch - mocksFromSecondCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromSecondFetchCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - mocksFromThirdCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - mocksFromThirdFetchCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - process.setClient(client); - - // start whole process - process.migrate(configurationFile); - - - - - assertThat(Input.fromReader(new StringReader((String) firstArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("k7notexists/k7batches/1_batch.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - writeDebFile((String) secondArg.getValue()); - - assertThat(Input.fromReader(new StringReader((String) secondArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("k7notexists/k7batches/2_batch.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - } - - private void writeDebFile(String str) { - try { - File f = new File("test.batch"); - f.createNewFile(); - IOUtils.write(str, new FileOutputStream(f)); - - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - @Test - public void testFullProcess_NOT_EXISTS_K7_TRANSFORM_SOLR_CLOUD() throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { - - InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("config.replicate.mlp_k7_solrcloud.xml"); - String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); - - InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("k7notexists/logfile_log"); - String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); - File logFile = File.createTempFile("junit","logfile"); - FileUtils.write(logFile, logFileContent, "UTF-8"); - - String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); - File configurationFile = File.createTempFile("junit","conf"); - FileUtils.write(configurationFile, configurationContent, "UTF-8"); - - - Client client = EasyMock.createMock(Client.class); - ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class) - .addMockedMethod("buildClient") - .createMock(); - - // --- 20 indexed --- - String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3Ae15e3a10-7774-11e7-89ee-5ef3fc9ae867%22+OR+%22uuid%3A420f4830-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Ae665ffa0-d6ad-11e7-8294-005056827e51%22+OR+%22uuid%3A88459bc0-cf86-11e7-a351-005056825209%22+OR+%22uuid%3A74bddf50-cf85-11e7-b059-5ef3fc9ae867%22+OR+%22uuid%3Ab7d82771-d69f-11e7-8294-005056827e51%22+OR+%22uuid%3A9f3f7bb0-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Af7217151-cf8a-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3A30bcc500-cf8f-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3Ae32eb050-d466-11e7-a047-005056825209%22+OR+%22uuid%3Ac39f12a0-d44a-11e7-a536-5ef3fc9ae867%22+OR+%22uuid%3Ab6632460-7641-11e4-9605-005056825209%22+OR+%22uuid%3A1e062920-d6be-11e7-9268-5ef3fc9ae867%22+OR+%22uuid%3Ab6c1a8f0-a730-11e2-b6da-005056827e52%22+OR+%22uuid%3A7c5e1e70-cc80-11e3-aec3-005056827e52%22+OR+%22uuid%3A8afd1150-1b3e-11e3-b62e-005056825209%22+OR+%22uuid%3A72356250-92c3-11e9-b601-005056825209%22+OR+%22uuid%3Aca058c00-929e-11e9-bc42-5ef3fc9ae867%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22)&fl=pid+collection+root.pid&wt=xml&rows=20"; - String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/1.xml"), "UTF-8"); - List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); - - String secondReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3Ae15e3a10-7774-11e7-89ee-5ef3fc9ae867%22+OR+%22uuid%3A420f4830-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Ae665ffa0-d6ad-11e7-8294-005056827e51%22+OR+%22uuid%3A88459bc0-cf86-11e7-a351-005056825209%22+OR+%22uuid%3A74bddf50-cf85-11e7-b059-5ef3fc9ae867%22+OR+%22uuid%3Ab7d82771-d69f-11e7-8294-005056827e51%22+OR+%22uuid%3A9f3f7bb0-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Af7217151-cf8a-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3A30bcc500-cf8f-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3Ae32eb050-d466-11e7-a047-005056825209%22+OR+%22uuid%3Ac39f12a0-d44a-11e7-a536-5ef3fc9ae867%22+OR+%22uuid%3Ab6632460-7641-11e4-9605-005056825209%22+OR+%22uuid%3A1e062920-d6be-11e7-9268-5ef3fc9ae867%22+OR+%22uuid%3Ab6c1a8f0-a730-11e2-b6da-005056827e52%22+OR+%22uuid%3A7c5e1e70-cc80-11e3-aec3-005056827e52%22+OR+%22uuid%3A8afd1150-1b3e-11e3-b62e-005056825209%22+OR+%22uuid%3A72356250-92c3-11e9-b601-005056825209%22+OR+%22uuid%3Aca058c00-929e-11e9-bc42-5ef3fc9ae867%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; - String secondRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/1_fetch.xml"), "UTF-8"); - List mocksFromSecondFetchCall = webCallExpect(client, secondReqFetch, secondRespFetch); - // ---------- - - // --- 3 indexed --- - String thirdReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3A3b624880-96b5-11e2-9a08-005056827e52%22+OR+%22uuid%3A87d2a060-a692-11e2-8b87-005056827e51%22+OR+%22uuid%3A653cfcf0-3fac-11e7-b56f-005056827e52%22)&fl=pid+collection+root.pid&wt=xml&rows=3"; - String thirdResp = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/1.xml"), "UTF-8"); - List mocksFromThirdCall = webCallExpect(client, thirdReq, thirdResp); - - String thirdReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A3b624880-96b5-11e2-9a08-005056827e52%22+OR+%22uuid%3A87d2a060-a692-11e2-8b87-005056827e51%22+OR+%22uuid%3A653cfcf0-3fac-11e7-b56f-005056827e52%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=3"; - String thirdRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/2_fetch.xml"), "UTF-8"); - List mocksFromThirdFetchCall = webCallExpect(client, thirdReqFetch, thirdRespFetch); - // ---------- - - - EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); - - // all updates in one check - WebResource updateResource = EasyMock.createMock(WebResource.class); - ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); - WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")).andReturn(updateResource).anyTimes(); - EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); - EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); - - //EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); - //Capture captureSingleArgument = EasyMock.newCa - Capture firstArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture secondArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(secondArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture thirdArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(thirdArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture fourthArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(fourthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - Capture fifthArg = newCapture(); - EasyMock.expect(updateResponseBuilder.entity(capture(fifthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); - - - EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); - EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); - EasyMock.expect(updateResponse.getEntityInputStream()).andDelegateTo(new MockClientResponse()).anyTimes(); - - // final commit - WebResource commitResource = EasyMock.createMock(WebResource.class); - WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); - - EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")).andReturn(commitResource).anyTimes(); - EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(commitResourceBuilder).anyTimes(); - EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); - - EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, - commitResource, - commitResourceBuilder); - - // check & fetch - mocksFromSecondCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - mocksFromSecondFetchCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - mocksFromThirdCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - mocksFromThirdFetchCall.stream().forEach(obj-> { - EasyMock.replay(obj); - }); - - process.setClient(client); - - // start whole process - process.migrate(configurationFile); - - - assertThat(Input.fromReader(new StringReader((String) firstArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("k7notexists/k7batches/1_batch_solrcloud.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - - writeDebFile((String) secondArg.getValue()); - - - assertThat(Input.fromReader(new StringReader((String) secondArg.getValue()))) - .and(Input.fromStream(this.getClass().getResourceAsStream("k7notexists/k7batches/2_batch_solrcloud.xml"))) - .ignoreComments() - .ignoreWhitespace() - .areSimilar(); - - } +// /** Simple copy index - data are copied */ +// @Test +// public void testFullProcess_NOT_EXISTS() throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { +// InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("config.replicate.mlp.xml"); +// String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); +// +// InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("logfile_log"); +// String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); +// File logFile = File.createTempFile("junit","logfile"); +// FileUtils.write(logFile, logFileContent, "UTF-8"); +// +// String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); +// File configurationFile = File.createTempFile("junit","conf"); +// FileUtils.write(configurationFile, configurationContent, "UTF-8"); +// +// +// Client client = EasyMock.createMock(Client.class); +// ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class) +// .addMockedMethod("buildClient") +// .createMock(); +// +// // --- 20 indexed --- +// String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A00000834-c06c-49ed-91e9-d54a0f8c8571%22+OR+%22uuid%3A00001fe4-1d09-42e2-845a-0201dd900e87%22+OR+%22uuid%3A0000211b-7c66-4730-b343-8ea2ef427f99%22+OR+%22uuid%3A000026fa-3695-4af4-aaaa-50557d8c2c6d%22+OR+%22uuid%3A0000329c-2940-89be-717b-82cac72948a8%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%401%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%402%22+OR+%22uuid%3A00003962-b232-4055-9a77-8d8db9487b90%22+OR+%22uuid%3A00003bb1-429d-11e2-849c-005056a60003%22+OR+%22uuid%3A000040ee-eda5-49d3-ae14-510a74921cda%22+OR+%22uuid%3A000047d7-3b67-4f08-a7cf-e843db917097%22+OR+%22uuid%3A000059eb-d782-4285-857a-41b370dc1afd%22+OR+%22uuid%3A00006174-e3da-488b-9a18-eef7b3d5e527%22+OR+%22uuid%3A00006bd9-100d-4c89-97e6-3529e99c45be%22+OR+%22uuid%3A00006e78-04c4-4fa0-9a9d-d7ba78f7c341%22+OR+%22uuid%3A00007ca2-054e-4fde-b73c-81f55a282ebc%22+OR+%22uuid%3A00007e8c-3f3a-4f73-ab70-0f95b1024f59%22+OR+%22uuid%3A000091fb-dc07-4594-9300-8a503dc09210%22+OR+%22uuid%3A0000a0c8-41b2-4fdd-9597-619b3a7ff965%22)&fl=PID+collection&wt=xml&rows=20"; +// String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/1.xml"), "UTF-8"); +// List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); +// +// String secondReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A00000834-c06c-49ed-91e9-d54a0f8c8571%22+OR+%22uuid%3A00001fe4-1d09-42e2-845a-0201dd900e87%22+OR+%22uuid%3A0000211b-7c66-4730-b343-8ea2ef427f99%22+OR+%22uuid%3A000026fa-3695-4af4-aaaa-50557d8c2c6d%22+OR+%22uuid%3A0000329c-2940-89be-717b-82cac72948a8%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%401%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%402%22+OR+%22uuid%3A00003962-b232-4055-9a77-8d8db9487b90%22+OR+%22uuid%3A00003bb1-429d-11e2-849c-005056a60003%22+OR+%22uuid%3A000040ee-eda5-49d3-ae14-510a74921cda%22+OR+%22uuid%3A000047d7-3b67-4f08-a7cf-e843db917097%22+OR+%22uuid%3A000059eb-d782-4285-857a-41b370dc1afd%22+OR+%22uuid%3A00006174-e3da-488b-9a18-eef7b3d5e527%22+OR+%22uuid%3A00006bd9-100d-4c89-97e6-3529e99c45be%22+OR+%22uuid%3A00006e78-04c4-4fa0-9a9d-d7ba78f7c341%22+OR+%22uuid%3A00007ca2-054e-4fde-b73c-81f55a282ebc%22+OR+%22uuid%3A00007e8c-3f3a-4f73-ab70-0f95b1024f59%22+OR+%22uuid%3A000091fb-dc07-4594-9300-8a503dc09210%22+OR+%22uuid%3A0000a0c8-41b2-4fdd-9597-619b3a7ff965%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; +// String secondRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/1_fetch.xml"), "UTF-8"); +// List mocksFromSecondFetchCall = webCallExpect(client, secondReqFetch, secondRespFetch); +// // ---------- +// +// // --- 40 indexed --- +// String thirdReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A0000a140-892d-48e3-851c-c0722578db50%22+OR+%22uuid%3A0000a210-40ad-48c6-a324-b4bd5fefe0f1%22+OR+%22uuid%3A0000a64f-d935-4260-acce-88833c115963%22+OR+%22uuid%3A0000ab19-c07d-41ac-9c6c-b03240271247%22+OR+%22uuid%3A0000abeb-c9ef-4ee9-8db1-536132bfac84%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%401%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%402%22+OR+%22uuid%3A0000b509-ed4f-447e-895f-d17d0d4ad0a3%22+OR+%22uuid%3A0000bbf9-0881-4e13-a793-457ed22bc58d%22+OR+%22uuid%3A0000c101-821b-4e21-95a3-13110119e10c%22+OR+%22uuid%3A0000c509-14e4-40b6-b555-f9ad46bce4c0%22+OR+%22uuid%3A0000c71a-aec6-4772-95f7-9412fd19f74c%22+OR+%22uuid%3A0000d5b8-44be-4473-b5f3-9e1f2437ce61%22+OR+%22uuid%3A0000dc41-0c83-4fe3-b013-081b57c6db8e%22+OR+%22uuid%3A0000dcd8-9a40-4b91-b3be-c7ffa05f6295%22+OR+%22uuid%3A0000dedc-cf17-11e1-acea-005056a60003%22+OR+%22uuid%3A0000e1cf-0621-4f8a-9bbc-cfdf4d25f096%22+OR+%22uuid%3A0000ed1b-31a8-4275-96fe-e6c80ef3e1b1%22+OR+%22uuid%3A0000ffd8-d89c-42db-a818-537144d80261%22)&fl=PID+collection&wt=xml&rows=20"; +// String thirdResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/2.xml"), "UTF-8"); +// List mocksFromThirdCall = webCallExpect(client, thirdReq, thirdResp); +// +// String thirdReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A0000a140-892d-48e3-851c-c0722578db50%22+OR+%22uuid%3A0000a210-40ad-48c6-a324-b4bd5fefe0f1%22+OR+%22uuid%3A0000a64f-d935-4260-acce-88833c115963%22+OR+%22uuid%3A0000ab19-c07d-41ac-9c6c-b03240271247%22+OR+%22uuid%3A0000abeb-c9ef-4ee9-8db1-536132bfac84%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%401%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%402%22+OR+%22uuid%3A0000b509-ed4f-447e-895f-d17d0d4ad0a3%22+OR+%22uuid%3A0000bbf9-0881-4e13-a793-457ed22bc58d%22+OR+%22uuid%3A0000c101-821b-4e21-95a3-13110119e10c%22+OR+%22uuid%3A0000c509-14e4-40b6-b555-f9ad46bce4c0%22+OR+%22uuid%3A0000c71a-aec6-4772-95f7-9412fd19f74c%22+OR+%22uuid%3A0000d5b8-44be-4473-b5f3-9e1f2437ce61%22+OR+%22uuid%3A0000dc41-0c83-4fe3-b013-081b57c6db8e%22+OR+%22uuid%3A0000dcd8-9a40-4b91-b3be-c7ffa05f6295%22+OR+%22uuid%3A0000dedc-cf17-11e1-acea-005056a60003%22+OR+%22uuid%3A0000e1cf-0621-4f8a-9bbc-cfdf4d25f096%22+OR+%22uuid%3A0000ed1b-31a8-4275-96fe-e6c80ef3e1b1%22+OR+%22uuid%3A0000ffd8-d89c-42db-a818-537144d80261%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; +// String thirdRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/2_fetch.xml"), "UTF-8"); +// List mocksFromThirdFetchCall = webCallExpect(client, thirdReqFetch, thirdRespFetch); +// // ---------- +// +// // --- 60 indexed --- +// String fourthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A00010fd5-d7c8-46fa-b87a-d87ca2aa84a5%22+OR+%22uuid%3A00011d48-9699-4b21-b000-9b0c34995b63%22+OR+%22uuid%3A00012082-f119-4370-9c0c-b6055137d6d8%22+OR+%22uuid%3A00012289-fae2-44a5-ba57-db7e5ab1159b%22+OR+%22uuid%3A0001257f-b598-4a0e-9c1f-8b199c115929%22+OR+%22uuid%3A00012835-2127-42b5-b598-6ece58276164%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%2F%401%22+OR+%22uuid%3A00013431-3c37-40db-9b4c-e87bbdd1d8eb%22+OR+%22uuid%3A00013618-6699-4721-b8d1-61b3a96bd969%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%2F%401%22+OR+%22uuid%3A00013afa-b718-42b1-b9bf-7f633b97c747%22+OR+%22uuid%3A00013d61-2317-43d2-9a69-39686e37c527%22+OR+%22uuid%3A00015940-ee76-415d-8169-797789406cff%22+OR+%22uuid%3A00016234-018a-4cce-90a4-5eaecd78122f%22+OR+%22uuid%3A00016cdf-2591-465d-bfd0-fece6ae08bf1%22+OR+%22uuid%3A0001714e-bbb4-4d7a-856a-c79aa618bfa2%22+OR+%22uuid%3A00017177-b182-46da-b88a-1377215389ea%22+OR+%22uuid%3A0001759e-cad0-467b-b61a-cfe1381a76f6%22)&fl=PID+collection&wt=xml&rows=20"; +// String fourthResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/3.xml"), "UTF-8"); +// List mocksFromFourthCall = webCallExpect(client, fourthReq, fourthResp); +// +// String fourthReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A00010fd5-d7c8-46fa-b87a-d87ca2aa84a5%22+OR+%22uuid%3A00011d48-9699-4b21-b000-9b0c34995b63%22+OR+%22uuid%3A00012082-f119-4370-9c0c-b6055137d6d8%22+OR+%22uuid%3A00012289-fae2-44a5-ba57-db7e5ab1159b%22+OR+%22uuid%3A0001257f-b598-4a0e-9c1f-8b199c115929%22+OR+%22uuid%3A00012835-2127-42b5-b598-6ece58276164%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%2F%401%22+OR+%22uuid%3A00013431-3c37-40db-9b4c-e87bbdd1d8eb%22+OR+%22uuid%3A00013618-6699-4721-b8d1-61b3a96bd969%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%2F%401%22+OR+%22uuid%3A00013afa-b718-42b1-b9bf-7f633b97c747%22+OR+%22uuid%3A00013d61-2317-43d2-9a69-39686e37c527%22+OR+%22uuid%3A00015940-ee76-415d-8169-797789406cff%22+OR+%22uuid%3A00016234-018a-4cce-90a4-5eaecd78122f%22+OR+%22uuid%3A00016cdf-2591-465d-bfd0-fece6ae08bf1%22+OR+%22uuid%3A0001714e-bbb4-4d7a-856a-c79aa618bfa2%22+OR+%22uuid%3A00017177-b182-46da-b88a-1377215389ea%22+OR+%22uuid%3A0001759e-cad0-467b-b61a-cfe1381a76f6%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; +// String fourthRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/3_fetch.xml"), "UTF-8"); +// List mocksFromFourthFetchCall = webCallExpect(client, fourthReqFetch, fourthRespFetch); +// // ---------- +// +// +// // --- 80 indexed --- +// String fifthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A000182c3-0215-4103-90ce-9f912a3caf02%22+OR+%22uuid%3A00018da8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001900d-e592-4ad4-8f61-21289fa0108c%22+OR+%22uuid%3A000199e3-6c01-4055-b620-d809be8c34c9%22+OR+%22uuid%3A00019d0e-ebcf-4aa0-ae46-e85403f867d0%22+OR+%22uuid%3A0001a28e-d40f-4caa-b99d-56b6b272be10%22+OR+%22uuid%3A0001b5ad-a2aa-43ea-912c-f472f7d0a786%22+OR+%22uuid%3A0001c4ba-bedf-4dbd-beb4-f87baf5bec3c%22+OR+%22uuid%3A0001c89d-0b86-448c-91c1-688f6882d548%22+OR+%22uuid%3A0001caa1-2fd5-4e0d-a74a-516286ee9414%22+OR+%22uuid%3A0001cd7d-446b-4696-b10d-a7115c6e4ed7%22+OR+%22uuid%3A0001cff1-9c7c-47ac-a2f0-340514a12852%22+OR+%22uuid%3A0001d02f-af3e-4d9d-9cae-31f1373aeffb%22+OR+%22uuid%3A0001dbc9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbca-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcb-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcc-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcd-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbce-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcf-3105-11e9-8847-005056a2b051%22)&fl=PID+collection&wt=xml&rows=20"; +// String fifthResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/4.xml"), "UTF-8"); +// List mocksFromFifthCall = webCallExpect(client, fifthReq, fifthResp); +// +// String fifthReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A000182c3-0215-4103-90ce-9f912a3caf02%22+OR+%22uuid%3A00018da8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001900d-e592-4ad4-8f61-21289fa0108c%22+OR+%22uuid%3A000199e3-6c01-4055-b620-d809be8c34c9%22+OR+%22uuid%3A00019d0e-ebcf-4aa0-ae46-e85403f867d0%22+OR+%22uuid%3A0001a28e-d40f-4caa-b99d-56b6b272be10%22+OR+%22uuid%3A0001b5ad-a2aa-43ea-912c-f472f7d0a786%22+OR+%22uuid%3A0001c4ba-bedf-4dbd-beb4-f87baf5bec3c%22+OR+%22uuid%3A0001c89d-0b86-448c-91c1-688f6882d548%22+OR+%22uuid%3A0001caa1-2fd5-4e0d-a74a-516286ee9414%22+OR+%22uuid%3A0001cd7d-446b-4696-b10d-a7115c6e4ed7%22+OR+%22uuid%3A0001cff1-9c7c-47ac-a2f0-340514a12852%22+OR+%22uuid%3A0001d02f-af3e-4d9d-9cae-31f1373aeffb%22+OR+%22uuid%3A0001dbc9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbca-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcb-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcc-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcd-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbce-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcf-3105-11e9-8847-005056a2b051%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; +// String fifthRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/4_fetch.xml"), "UTF-8"); +// List mocksFromFifthFetchCall = webCallExpect(client, fifthReqFetch, fifthRespFetch); +// // ---------- +// +// +// // --- 100 indexed --- +// String sixthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A0001dbd0-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001de4b-75ee-4428-bb20-737cc2cee698%22+OR+%22uuid%3A0001e211-3e75-4c7e-871d-662c2bfc4471%22+OR+%22uuid%3A0001e71d-9c40-482c-b0b4-70fa1d62d4ed%22+OR+%22uuid%3A0001e8ae-02ab-49e7-bb84-d83291bc075e%22+OR+%22uuid%3A0001eff8-4618-417a-aede-4ab6b5274f8c%22+OR+%22uuid%3A0001f258-7472-42df-a994-c14817e8e9c5%22+OR+%22uuid%3A0001f761-5a0c-47cc-87de-db9513e4c305%22+OR+%22uuid%3A000202e1-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e2-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e3-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e4-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e5-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e6-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e7-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A00020759-510b-429d-a46c-ed0b25eb4a02%22+OR+%22uuid%3A000229fa-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000229fb-3105-11e9-8847-005056a2b051%22)&fl=PID+collection&wt=xml&rows=20"; +// String sixthResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/5.xml"), "UTF-8"); +// List mocksFromSixthCall = webCallExpect(client, sixthReq, sixthResp); +// +// String sixthReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A0001dbd0-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001de4b-75ee-4428-bb20-737cc2cee698%22+OR+%22uuid%3A0001e211-3e75-4c7e-871d-662c2bfc4471%22+OR+%22uuid%3A0001e71d-9c40-482c-b0b4-70fa1d62d4ed%22+OR+%22uuid%3A0001e8ae-02ab-49e7-bb84-d83291bc075e%22+OR+%22uuid%3A0001eff8-4618-417a-aede-4ab6b5274f8c%22+OR+%22uuid%3A0001f258-7472-42df-a994-c14817e8e9c5%22+OR+%22uuid%3A0001f761-5a0c-47cc-87de-db9513e4c305%22+OR+%22uuid%3A000202e1-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e2-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e3-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e4-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e5-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e6-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e7-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A00020759-510b-429d-a46c-ed0b25eb4a02%22+OR+%22uuid%3A000229fa-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000229fb-3105-11e9-8847-005056a2b051%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; +// String sixthRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/5_fetch.xml"), "UTF-8"); +// List mocksFromSixthFetchCall = webCallExpect(client, sixthReqFetch, sixthRespFetch); +// // ---------- +// +// EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); +// +// // all updates in one check +// WebResource updateResource = EasyMock.createMock(WebResource.class); +// ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); +// WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); +// EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")).andReturn(updateResource).anyTimes(); +// EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); +// EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); +// +// //EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); +// //Capture captureSingleArgument = EasyMock.newCa +// Capture firstArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture secondArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(secondArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture thirdArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(thirdArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture fourthArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(fourthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture fifthArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(fifthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// +// EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); +// EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); +// EasyMock.expect(updateResponse.getEntityInputStream()).andDelegateTo(new MockClientResponse()).anyTimes(); +// +// // final commit +// WebResource commitResource = EasyMock.createMock(WebResource.class); +// WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); +// +// EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")).andReturn(commitResource).anyTimes(); +// EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); +// +// EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, +// commitResource, +// commitResourceBuilder); +// +// // check & fetch +// mocksFromSecondCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromSecondFetchCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// // check & fetch +// mocksFromThirdCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromThirdFetchCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// // check & fetch +// mocksFromFourthCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromFourthFetchCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// // check & fetch +// mocksFromFifthCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromFifthFetchCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// +// // check & fetch +// mocksFromSixthCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromSixthFetchCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// process.setClient(client); +// +// // start whole process +// process.migrate(configurationFile); +// +// +// +// assertThat(Input.fromReader(new StringReader((String) firstArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/1_batch.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// assertThat(Input.fromReader(new StringReader((String) secondArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/2_batch.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// assertThat(Input.fromReader(new StringReader((String) thirdArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/3_batch.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// assertThat(Input.fromReader(new StringReader((String) fourthArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/4_batch.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// assertThat(Input.fromReader(new StringReader((String) fifthArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/5_batch.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// +// } +// +// +// /** Simple copy index - data are updated but in solr cloud */ +// @Test +// public void testFullProcess_EXISTS_SOLR_CLOUD() throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { +// InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("config.replicate.mlp_solrcloud.xml"); +// String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); +// +// InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("logfile_log"); +// String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); +// File logFile = File.createTempFile("junit","logfile"); +// FileUtils.write(logFile, logFileContent, "UTF-8"); +// +// String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); +// File configurationFile = File.createTempFile("junit","conf"); +// FileUtils.write(configurationFile, configurationContent, "UTF-8"); +// +// +// Client client = EasyMock.createMock(Client.class); +// ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class) +// .addMockedMethod("buildClient") +// .createMock(); +// +// // --- 20 updated --- +// String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A00000834-c06c-49ed-91e9-d54a0f8c8571%22+OR+%22uuid%3A00001fe4-1d09-42e2-845a-0201dd900e87%22+OR+%22uuid%3A0000211b-7c66-4730-b343-8ea2ef427f99%22+OR+%22uuid%3A000026fa-3695-4af4-aaaa-50557d8c2c6d%22+OR+%22uuid%3A0000329c-2940-89be-717b-82cac72948a8%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%401%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%402%22+OR+%22uuid%3A00003962-b232-4055-9a77-8d8db9487b90%22+OR+%22uuid%3A00003bb1-429d-11e2-849c-005056a60003%22+OR+%22uuid%3A000040ee-eda5-49d3-ae14-510a74921cda%22+OR+%22uuid%3A000047d7-3b67-4f08-a7cf-e843db917097%22+OR+%22uuid%3A000059eb-d782-4285-857a-41b370dc1afd%22+OR+%22uuid%3A00006174-e3da-488b-9a18-eef7b3d5e527%22+OR+%22uuid%3A00006bd9-100d-4c89-97e6-3529e99c45be%22+OR+%22uuid%3A00006e78-04c4-4fa0-9a9d-d7ba78f7c341%22+OR+%22uuid%3A00007ca2-054e-4fde-b73c-81f55a282ebc%22+OR+%22uuid%3A00007e8c-3f3a-4f73-ab70-0f95b1024f59%22+OR+%22uuid%3A000091fb-dc07-4594-9300-8a503dc09210%22+OR+%22uuid%3A0000a0c8-41b2-4fdd-9597-619b3a7ff965%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; +// String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("exists/1_solrcloud.xml"), "UTF-8"); +// List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); +// // ---------- +// +// // --- 40 updated --- +// String thirdReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A0000a140-892d-48e3-851c-c0722578db50%22+OR+%22uuid%3A0000a210-40ad-48c6-a324-b4bd5fefe0f1%22+OR+%22uuid%3A0000a64f-d935-4260-acce-88833c115963%22+OR+%22uuid%3A0000ab19-c07d-41ac-9c6c-b03240271247%22+OR+%22uuid%3A0000abeb-c9ef-4ee9-8db1-536132bfac84%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%401%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%402%22+OR+%22uuid%3A0000b509-ed4f-447e-895f-d17d0d4ad0a3%22+OR+%22uuid%3A0000bbf9-0881-4e13-a793-457ed22bc58d%22+OR+%22uuid%3A0000c101-821b-4e21-95a3-13110119e10c%22+OR+%22uuid%3A0000c509-14e4-40b6-b555-f9ad46bce4c0%22+OR+%22uuid%3A0000c71a-aec6-4772-95f7-9412fd19f74c%22+OR+%22uuid%3A0000d5b8-44be-4473-b5f3-9e1f2437ce61%22+OR+%22uuid%3A0000dc41-0c83-4fe3-b013-081b57c6db8e%22+OR+%22uuid%3A0000dcd8-9a40-4b91-b3be-c7ffa05f6295%22+OR+%22uuid%3A0000dedc-cf17-11e1-acea-005056a60003%22+OR+%22uuid%3A0000e1cf-0621-4f8a-9bbc-cfdf4d25f096%22+OR+%22uuid%3A0000ed1b-31a8-4275-96fe-e6c80ef3e1b1%22+OR+%22uuid%3A0000ffd8-d89c-42db-a818-537144d80261%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; +// String thirdResp = IOUtils.toString(this.getClass().getResourceAsStream("exists/2_solrcloud.xml"), "UTF-8"); +// List mocksFromThirdCall = webCallExpect(client, thirdReq, thirdResp); +// // ---------- +// +// // --- 60 updated --- +// String fourthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A00010fd5-d7c8-46fa-b87a-d87ca2aa84a5%22+OR+%22uuid%3A00011d48-9699-4b21-b000-9b0c34995b63%22+OR+%22uuid%3A00012082-f119-4370-9c0c-b6055137d6d8%22+OR+%22uuid%3A00012289-fae2-44a5-ba57-db7e5ab1159b%22+OR+%22uuid%3A0001257f-b598-4a0e-9c1f-8b199c115929%22+OR+%22uuid%3A00012835-2127-42b5-b598-6ece58276164%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%2F%401%22+OR+%22uuid%3A00013431-3c37-40db-9b4c-e87bbdd1d8eb%22+OR+%22uuid%3A00013618-6699-4721-b8d1-61b3a96bd969%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%2F%401%22+OR+%22uuid%3A00013afa-b718-42b1-b9bf-7f633b97c747%22+OR+%22uuid%3A00013d61-2317-43d2-9a69-39686e37c527%22+OR+%22uuid%3A00015940-ee76-415d-8169-797789406cff%22+OR+%22uuid%3A00016234-018a-4cce-90a4-5eaecd78122f%22+OR+%22uuid%3A00016cdf-2591-465d-bfd0-fece6ae08bf1%22+OR+%22uuid%3A0001714e-bbb4-4d7a-856a-c79aa618bfa2%22+OR+%22uuid%3A00017177-b182-46da-b88a-1377215389ea%22+OR+%22uuid%3A0001759e-cad0-467b-b61a-cfe1381a76f6%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; +// String fourthResp = IOUtils.toString(this.getClass().getResourceAsStream("exists/3_solrcloud.xml"), "UTF-8"); +// List mocksFromFourthCall = webCallExpect(client, fourthReq, fourthResp); +// // ---------- +// +// // --- 80 updated --- +// String fifthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A000182c3-0215-4103-90ce-9f912a3caf02%22+OR+%22uuid%3A00018da8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001900d-e592-4ad4-8f61-21289fa0108c%22+OR+%22uuid%3A000199e3-6c01-4055-b620-d809be8c34c9%22+OR+%22uuid%3A00019d0e-ebcf-4aa0-ae46-e85403f867d0%22+OR+%22uuid%3A0001a28e-d40f-4caa-b99d-56b6b272be10%22+OR+%22uuid%3A0001b5ad-a2aa-43ea-912c-f472f7d0a786%22+OR+%22uuid%3A0001c4ba-bedf-4dbd-beb4-f87baf5bec3c%22+OR+%22uuid%3A0001c89d-0b86-448c-91c1-688f6882d548%22+OR+%22uuid%3A0001caa1-2fd5-4e0d-a74a-516286ee9414%22+OR+%22uuid%3A0001cd7d-446b-4696-b10d-a7115c6e4ed7%22+OR+%22uuid%3A0001cff1-9c7c-47ac-a2f0-340514a12852%22+OR+%22uuid%3A0001d02f-af3e-4d9d-9cae-31f1373aeffb%22+OR+%22uuid%3A0001dbc9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbca-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcb-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcc-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcd-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbce-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcf-3105-11e9-8847-005056a2b051%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; +// String fifthResp = IOUtils.toString(this.getClass().getResourceAsStream("exists/4_solrcloud.xml"), "UTF-8"); +// List mocksFromFifthCall = webCallExpect(client, fifthReq, fifthResp); +// // ---------- +// +// // --- 100 updated --- +// String sixthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A0001dbd0-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001de4b-75ee-4428-bb20-737cc2cee698%22+OR+%22uuid%3A0001e211-3e75-4c7e-871d-662c2bfc4471%22+OR+%22uuid%3A0001e71d-9c40-482c-b0b4-70fa1d62d4ed%22+OR+%22uuid%3A0001e8ae-02ab-49e7-bb84-d83291bc075e%22+OR+%22uuid%3A0001eff8-4618-417a-aede-4ab6b5274f8c%22+OR+%22uuid%3A0001f258-7472-42df-a994-c14817e8e9c5%22+OR+%22uuid%3A0001f761-5a0c-47cc-87de-db9513e4c305%22+OR+%22uuid%3A000202e1-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e2-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e3-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e4-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e5-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e6-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e7-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A00020759-510b-429d-a46c-ed0b25eb4a02%22+OR+%22uuid%3A000229fa-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000229fb-3105-11e9-8847-005056a2b051%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; +// String sixthResp = IOUtils.toString(this.getClass().getResourceAsStream("exists/5_solrcloud.xml"), "UTF-8"); +// List mocksFromSixthCall = webCallExpect(client, sixthReq, sixthResp); +// // ---------- +// +// +// EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); +// +// // all updates in one check; (anyString) +// WebResource updateResource = EasyMock.createMock(WebResource.class); +// ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); +// WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); +// EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")).andReturn(updateResource).anyTimes(); +// EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); +// EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); +// +// +// Capture firstArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture secondArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(secondArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture thirdArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(thirdArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture fourthArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(fourthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture fifthArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(fifthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// +// //EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); +// +// EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); +// EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); +// EasyMock.expect(updateResponse.getEntityInputStream()).andDelegateTo(new MockClientResponse()).anyTimes(); +// +// // final commit +// +// WebResource commitResource = EasyMock.createMock(WebResource.class); +// WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); +// +// EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")).andReturn(commitResource).anyTimes(); +// EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); +// +// EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, +// commitResource, +// commitResourceBuilder); +// mocksFromSecondCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromThirdCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromFourthCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromFifthCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromSixthCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// process.setClient(client); +// +// // start whole process +// process.migrate(configurationFile); +// +// assertThat(Input.fromReader(new StringReader((String) firstArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("exists/batches/1_batch_solrcloud.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// assertThat(Input.fromReader(new StringReader((String) secondArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("exists/batches/2_batch_solrcloud.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// assertThat(Input.fromReader(new StringReader((String) thirdArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("exists/batches/3_batch_solrcloud.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// assertThat(Input.fromReader(new StringReader((String) fourthArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("exists/batches/4_batch_solrcloud.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// assertThat(Input.fromReader(new StringReader((String) fifthArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("exists/batches/5_batch_solrcloud.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// } +// +// +// +// /** Simple copy index - data are indexed byt in solr cloud */ +// @Test +// public void testFullProcess_NOT_EXISTS_SOLR_CLOUD() throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { +// InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("config.replicate.mlp_solrcloud.xml"); +// String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); +// +// InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("logfile_log"); +// String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); +// File logFile = File.createTempFile("junit","logfile"); +// FileUtils.write(logFile, logFileContent, "UTF-8"); +// +// String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); +// File configurationFile = File.createTempFile("junit","conf"); +// FileUtils.write(configurationFile, configurationContent, "UTF-8"); +// +// Client client = EasyMock.createMock(Client.class); +// ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class) +// .addMockedMethod("buildClient") +// .createMock(); +// +// // --- 20 indexed --- +// String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A00000834-c06c-49ed-91e9-d54a0f8c8571%22+OR+%22uuid%3A00001fe4-1d09-42e2-845a-0201dd900e87%22+OR+%22uuid%3A0000211b-7c66-4730-b343-8ea2ef427f99%22+OR+%22uuid%3A000026fa-3695-4af4-aaaa-50557d8c2c6d%22+OR+%22uuid%3A0000329c-2940-89be-717b-82cac72948a8%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%401%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%402%22+OR+%22uuid%3A00003962-b232-4055-9a77-8d8db9487b90%22+OR+%22uuid%3A00003bb1-429d-11e2-849c-005056a60003%22+OR+%22uuid%3A000040ee-eda5-49d3-ae14-510a74921cda%22+OR+%22uuid%3A000047d7-3b67-4f08-a7cf-e843db917097%22+OR+%22uuid%3A000059eb-d782-4285-857a-41b370dc1afd%22+OR+%22uuid%3A00006174-e3da-488b-9a18-eef7b3d5e527%22+OR+%22uuid%3A00006bd9-100d-4c89-97e6-3529e99c45be%22+OR+%22uuid%3A00006e78-04c4-4fa0-9a9d-d7ba78f7c341%22+OR+%22uuid%3A00007ca2-054e-4fde-b73c-81f55a282ebc%22+OR+%22uuid%3A00007e8c-3f3a-4f73-ab70-0f95b1024f59%22+OR+%22uuid%3A000091fb-dc07-4594-9300-8a503dc09210%22+OR+%22uuid%3A0000a0c8-41b2-4fdd-9597-619b3a7ff965%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; +// String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/1.xml"), "UTF-8"); +// List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); +// +// String secondReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A00000834-c06c-49ed-91e9-d54a0f8c8571%22+OR+%22uuid%3A00001fe4-1d09-42e2-845a-0201dd900e87%22+OR+%22uuid%3A0000211b-7c66-4730-b343-8ea2ef427f99%22+OR+%22uuid%3A000026fa-3695-4af4-aaaa-50557d8c2c6d%22+OR+%22uuid%3A0000329c-2940-89be-717b-82cac72948a8%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%401%22+OR+%22uuid%3A0000338c-4dd1-4d11-bda5-4c1b44d83628%2F%402%22+OR+%22uuid%3A00003962-b232-4055-9a77-8d8db9487b90%22+OR+%22uuid%3A00003bb1-429d-11e2-849c-005056a60003%22+OR+%22uuid%3A000040ee-eda5-49d3-ae14-510a74921cda%22+OR+%22uuid%3A000047d7-3b67-4f08-a7cf-e843db917097%22+OR+%22uuid%3A000059eb-d782-4285-857a-41b370dc1afd%22+OR+%22uuid%3A00006174-e3da-488b-9a18-eef7b3d5e527%22+OR+%22uuid%3A00006bd9-100d-4c89-97e6-3529e99c45be%22+OR+%22uuid%3A00006e78-04c4-4fa0-9a9d-d7ba78f7c341%22+OR+%22uuid%3A00007ca2-054e-4fde-b73c-81f55a282ebc%22+OR+%22uuid%3A00007e8c-3f3a-4f73-ab70-0f95b1024f59%22+OR+%22uuid%3A000091fb-dc07-4594-9300-8a503dc09210%22+OR+%22uuid%3A0000a0c8-41b2-4fdd-9597-619b3a7ff965%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; +// String secondRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/1_fetch.xml"), "UTF-8"); +// List mocksFromSecondFetchCall = webCallExpect(client, secondReqFetch, secondRespFetch); +// // ---------- +// +// // --- 40 indexed --- +// String thirdReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A0000a140-892d-48e3-851c-c0722578db50%22+OR+%22uuid%3A0000a210-40ad-48c6-a324-b4bd5fefe0f1%22+OR+%22uuid%3A0000a64f-d935-4260-acce-88833c115963%22+OR+%22uuid%3A0000ab19-c07d-41ac-9c6c-b03240271247%22+OR+%22uuid%3A0000abeb-c9ef-4ee9-8db1-536132bfac84%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%401%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%402%22+OR+%22uuid%3A0000b509-ed4f-447e-895f-d17d0d4ad0a3%22+OR+%22uuid%3A0000bbf9-0881-4e13-a793-457ed22bc58d%22+OR+%22uuid%3A0000c101-821b-4e21-95a3-13110119e10c%22+OR+%22uuid%3A0000c509-14e4-40b6-b555-f9ad46bce4c0%22+OR+%22uuid%3A0000c71a-aec6-4772-95f7-9412fd19f74c%22+OR+%22uuid%3A0000d5b8-44be-4473-b5f3-9e1f2437ce61%22+OR+%22uuid%3A0000dc41-0c83-4fe3-b013-081b57c6db8e%22+OR+%22uuid%3A0000dcd8-9a40-4b91-b3be-c7ffa05f6295%22+OR+%22uuid%3A0000dedc-cf17-11e1-acea-005056a60003%22+OR+%22uuid%3A0000e1cf-0621-4f8a-9bbc-cfdf4d25f096%22+OR+%22uuid%3A0000ed1b-31a8-4275-96fe-e6c80ef3e1b1%22+OR+%22uuid%3A0000ffd8-d89c-42db-a818-537144d80261%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; +// String thirdResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/2.xml"), "UTF-8"); +// List mocksFromThirdCall = webCallExpect(client, thirdReq, thirdResp); +// +// String thirdReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A0000a140-892d-48e3-851c-c0722578db50%22+OR+%22uuid%3A0000a210-40ad-48c6-a324-b4bd5fefe0f1%22+OR+%22uuid%3A0000a64f-d935-4260-acce-88833c115963%22+OR+%22uuid%3A0000ab19-c07d-41ac-9c6c-b03240271247%22+OR+%22uuid%3A0000abeb-c9ef-4ee9-8db1-536132bfac84%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%401%22+OR+%22uuid%3A0000b3cd-478a-4756-b302-31765cab9efc%2F%402%22+OR+%22uuid%3A0000b509-ed4f-447e-895f-d17d0d4ad0a3%22+OR+%22uuid%3A0000bbf9-0881-4e13-a793-457ed22bc58d%22+OR+%22uuid%3A0000c101-821b-4e21-95a3-13110119e10c%22+OR+%22uuid%3A0000c509-14e4-40b6-b555-f9ad46bce4c0%22+OR+%22uuid%3A0000c71a-aec6-4772-95f7-9412fd19f74c%22+OR+%22uuid%3A0000d5b8-44be-4473-b5f3-9e1f2437ce61%22+OR+%22uuid%3A0000dc41-0c83-4fe3-b013-081b57c6db8e%22+OR+%22uuid%3A0000dcd8-9a40-4b91-b3be-c7ffa05f6295%22+OR+%22uuid%3A0000dedc-cf17-11e1-acea-005056a60003%22+OR+%22uuid%3A0000e1cf-0621-4f8a-9bbc-cfdf4d25f096%22+OR+%22uuid%3A0000ed1b-31a8-4275-96fe-e6c80ef3e1b1%22+OR+%22uuid%3A0000ffd8-d89c-42db-a818-537144d80261%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; +// String thirdRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/2_fetch.xml"), "UTF-8"); +// List mocksFromThirdFetchCall = webCallExpect(client, thirdReqFetch, thirdRespFetch); +// // ---------- +// +// // --- 60 indexed --- +// String fourthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A00010fd5-d7c8-46fa-b87a-d87ca2aa84a5%22+OR+%22uuid%3A00011d48-9699-4b21-b000-9b0c34995b63%22+OR+%22uuid%3A00012082-f119-4370-9c0c-b6055137d6d8%22+OR+%22uuid%3A00012289-fae2-44a5-ba57-db7e5ab1159b%22+OR+%22uuid%3A0001257f-b598-4a0e-9c1f-8b199c115929%22+OR+%22uuid%3A00012835-2127-42b5-b598-6ece58276164%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%2F%401%22+OR+%22uuid%3A00013431-3c37-40db-9b4c-e87bbdd1d8eb%22+OR+%22uuid%3A00013618-6699-4721-b8d1-61b3a96bd969%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%2F%401%22+OR+%22uuid%3A00013afa-b718-42b1-b9bf-7f633b97c747%22+OR+%22uuid%3A00013d61-2317-43d2-9a69-39686e37c527%22+OR+%22uuid%3A00015940-ee76-415d-8169-797789406cff%22+OR+%22uuid%3A00016234-018a-4cce-90a4-5eaecd78122f%22+OR+%22uuid%3A00016cdf-2591-465d-bfd0-fece6ae08bf1%22+OR+%22uuid%3A0001714e-bbb4-4d7a-856a-c79aa618bfa2%22+OR+%22uuid%3A00017177-b182-46da-b88a-1377215389ea%22+OR+%22uuid%3A0001759e-cad0-467b-b61a-cfe1381a76f6%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; +// String fourthResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/3.xml"), "UTF-8"); +// List mocksFromFourthCall = webCallExpect(client, fourthReq, fourthResp); +// +// String fourthReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A00010fd5-d7c8-46fa-b87a-d87ca2aa84a5%22+OR+%22uuid%3A00011d48-9699-4b21-b000-9b0c34995b63%22+OR+%22uuid%3A00012082-f119-4370-9c0c-b6055137d6d8%22+OR+%22uuid%3A00012289-fae2-44a5-ba57-db7e5ab1159b%22+OR+%22uuid%3A0001257f-b598-4a0e-9c1f-8b199c115929%22+OR+%22uuid%3A00012835-2127-42b5-b598-6ece58276164%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%22+OR+%22uuid%3A00012ff0-a588-470b-9428-f2e1eddc2e6a%2F%401%22+OR+%22uuid%3A00013431-3c37-40db-9b4c-e87bbdd1d8eb%22+OR+%22uuid%3A00013618-6699-4721-b8d1-61b3a96bd969%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%22+OR+%22uuid%3A000136ec-ae05-426b-85e1-e6b38576f761%2F%401%22+OR+%22uuid%3A00013afa-b718-42b1-b9bf-7f633b97c747%22+OR+%22uuid%3A00013d61-2317-43d2-9a69-39686e37c527%22+OR+%22uuid%3A00015940-ee76-415d-8169-797789406cff%22+OR+%22uuid%3A00016234-018a-4cce-90a4-5eaecd78122f%22+OR+%22uuid%3A00016cdf-2591-465d-bfd0-fece6ae08bf1%22+OR+%22uuid%3A0001714e-bbb4-4d7a-856a-c79aa618bfa2%22+OR+%22uuid%3A00017177-b182-46da-b88a-1377215389ea%22+OR+%22uuid%3A0001759e-cad0-467b-b61a-cfe1381a76f6%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; +// String fourthRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/3_fetch.xml"), "UTF-8"); +// List mocksFromFourthFetchCall = webCallExpect(client, fourthReqFetch, fourthRespFetch); +// // ---------- +// +// +// // --- 80 indexed --- +// String fifthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A000182c3-0215-4103-90ce-9f912a3caf02%22+OR+%22uuid%3A00018da8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001900d-e592-4ad4-8f61-21289fa0108c%22+OR+%22uuid%3A000199e3-6c01-4055-b620-d809be8c34c9%22+OR+%22uuid%3A00019d0e-ebcf-4aa0-ae46-e85403f867d0%22+OR+%22uuid%3A0001a28e-d40f-4caa-b99d-56b6b272be10%22+OR+%22uuid%3A0001b5ad-a2aa-43ea-912c-f472f7d0a786%22+OR+%22uuid%3A0001c4ba-bedf-4dbd-beb4-f87baf5bec3c%22+OR+%22uuid%3A0001c89d-0b86-448c-91c1-688f6882d548%22+OR+%22uuid%3A0001caa1-2fd5-4e0d-a74a-516286ee9414%22+OR+%22uuid%3A0001cd7d-446b-4696-b10d-a7115c6e4ed7%22+OR+%22uuid%3A0001cff1-9c7c-47ac-a2f0-340514a12852%22+OR+%22uuid%3A0001d02f-af3e-4d9d-9cae-31f1373aeffb%22+OR+%22uuid%3A0001dbc9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbca-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcb-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcc-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcd-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbce-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcf-3105-11e9-8847-005056a2b051%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; +// String fifthResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/4.xml"), "UTF-8"); +// List mocksFromFifthCall = webCallExpect(client, fifthReq, fifthResp); +// +// String fifthReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A000182c3-0215-4103-90ce-9f912a3caf02%22+OR+%22uuid%3A00018da8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001900d-e592-4ad4-8f61-21289fa0108c%22+OR+%22uuid%3A000199e3-6c01-4055-b620-d809be8c34c9%22+OR+%22uuid%3A00019d0e-ebcf-4aa0-ae46-e85403f867d0%22+OR+%22uuid%3A0001a28e-d40f-4caa-b99d-56b6b272be10%22+OR+%22uuid%3A0001b5ad-a2aa-43ea-912c-f472f7d0a786%22+OR+%22uuid%3A0001c4ba-bedf-4dbd-beb4-f87baf5bec3c%22+OR+%22uuid%3A0001c89d-0b86-448c-91c1-688f6882d548%22+OR+%22uuid%3A0001caa1-2fd5-4e0d-a74a-516286ee9414%22+OR+%22uuid%3A0001cd7d-446b-4696-b10d-a7115c6e4ed7%22+OR+%22uuid%3A0001cff1-9c7c-47ac-a2f0-340514a12852%22+OR+%22uuid%3A0001d02f-af3e-4d9d-9cae-31f1373aeffb%22+OR+%22uuid%3A0001dbc9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbca-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcb-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcc-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcd-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbce-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001dbcf-3105-11e9-8847-005056a2b051%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; +// String fifthRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/4_fetch.xml"), "UTF-8"); +// List mocksFromFifthFetchCall = webCallExpect(client, fifthReqFetch, fifthRespFetch); +// // ---------- +// +// +// // --- 100 indexed --- +// String sixthReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=PID:(%22uuid%3A0001dbd0-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001de4b-75ee-4428-bb20-737cc2cee698%22+OR+%22uuid%3A0001e211-3e75-4c7e-871d-662c2bfc4471%22+OR+%22uuid%3A0001e71d-9c40-482c-b0b4-70fa1d62d4ed%22+OR+%22uuid%3A0001e8ae-02ab-49e7-bb84-d83291bc075e%22+OR+%22uuid%3A0001eff8-4618-417a-aede-4ab6b5274f8c%22+OR+%22uuid%3A0001f258-7472-42df-a994-c14817e8e9c5%22+OR+%22uuid%3A0001f761-5a0c-47cc-87de-db9513e4c305%22+OR+%22uuid%3A000202e1-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e2-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e3-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e4-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e5-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e6-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e7-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A00020759-510b-429d-a46c-ed0b25eb4a02%22+OR+%22uuid%3A000229fa-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000229fb-3105-11e9-8847-005056a2b051%22)&fl=PID+collection+root_pid&wt=xml&rows=20"; +// String sixthResp = IOUtils.toString(this.getClass().getResourceAsStream("notexists/5.xml"), "UTF-8"); +// List mocksFromSixthCall = webCallExpect(client, sixthReq, sixthResp); +// +// String sixthReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A0001dbd0-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A0001de4b-75ee-4428-bb20-737cc2cee698%22+OR+%22uuid%3A0001e211-3e75-4c7e-871d-662c2bfc4471%22+OR+%22uuid%3A0001e71d-9c40-482c-b0b4-70fa1d62d4ed%22+OR+%22uuid%3A0001e8ae-02ab-49e7-bb84-d83291bc075e%22+OR+%22uuid%3A0001eff8-4618-417a-aede-4ab6b5274f8c%22+OR+%22uuid%3A0001f258-7472-42df-a994-c14817e8e9c5%22+OR+%22uuid%3A0001f761-5a0c-47cc-87de-db9513e4c305%22+OR+%22uuid%3A000202e1-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e2-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e3-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e4-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e5-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e6-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e7-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e8-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000202e9-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A00020759-510b-429d-a46c-ed0b25eb4a02%22+OR+%22uuid%3A000229fa-3105-11e9-8847-005056a2b051%22+OR+%22uuid%3A000229fb-3105-11e9-8847-005056a2b051%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; +// String sixthRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("notexists/5_fetch.xml"), "UTF-8"); +// List mocksFromSixthFetchCall = webCallExpect(client, sixthReqFetch, sixthRespFetch); +// // ---------- +// +// EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); +// +// // all updates in one check +// WebResource updateResource = EasyMock.createMock(WebResource.class); +// ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); +// WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); +// EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")).andReturn(updateResource).anyTimes(); +// EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); +// EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); +// +// Capture firstArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture secondArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(secondArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture thirdArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(thirdArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture fourthArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(fourthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture fifthArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(fifthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// //EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); +// +// EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); +// EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); +// EasyMock.expect(updateResponse.getEntityInputStream()).andDelegateTo(new MockClientResponse()).anyTimes(); +// +// // final commit +// WebResource commitResource = EasyMock.createMock(WebResource.class); +// WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); +// +// EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")).andReturn(commitResource).anyTimes(); +// EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); +// +// EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, +// commitResource, +// commitResourceBuilder); +// +// // check & fetch +// mocksFromSecondCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromSecondFetchCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// // check & fetch +// mocksFromThirdCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromThirdFetchCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// // check & fetch +// mocksFromFourthCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromFourthFetchCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// // check & fetch +// mocksFromFifthCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromFifthFetchCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// +// // check & fetch +// mocksFromSixthCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromSixthFetchCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// process.setClient(client); +// +// // start whole process +// process.migrate(configurationFile); +// +// +// assertThat(Input.fromReader(new StringReader((String) firstArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/1_batch_solrcloud.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// assertThat(Input.fromReader(new StringReader((String) secondArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/2_batch_solrcloud.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// assertThat(Input.fromReader(new StringReader((String) thirdArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/3_batch_solrcloud.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// assertThat(Input.fromReader(new StringReader((String) fourthArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/4_batch_solrcloud.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// assertThat(Input.fromReader(new StringReader((String) fifthArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("notexists/batches/5_batch_solrcloud.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// +// } +// +// /** Copy and transform index K5->K7 - data are updated */ +// @Test +// public void testFullProcess_EXISTS_K7_TRANSFORM() throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { +// +// InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("config.replicate.mlp_k7.xml"); +// String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); +// +// InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("k7exists/logfile_log"); +// String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); +// File logFile = File.createTempFile("junit","logfile"); +// FileUtils.write(logFile, logFileContent, "UTF-8"); +// +// String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); +// File configurationFile = File.createTempFile("junit","conf"); +// FileUtils.write(configurationFile, configurationContent, "UTF-8"); +// +// +// Client client = EasyMock.createMock(Client.class); +// ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class) +// .addMockedMethod("buildClient") +// .createMock(); +// +// // --- 20 indexed --- +// String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3A03173900-662a-11e3-9ea2-5ef3fc9ae867%22+OR+%22uuid%3A03173e90-9c18-11e3-a880-5ef3fc9ae867%22+OR+%22uuid%3A03218150-2e9e-11eb-bd18-005056827e51%22+OR+%22uuid%3A03218050-35fe-11e4-8e0d-005056827e51%22+OR+%22uuid%3A03174110-5186-11e3-8c6a-005056825209%22+OR+%22uuid%3A03174020-f2fa-11e6-bf7e-005056825209%22+OR+%22uuid%3A03173f70-5dc0-11e9-a9b4-5ef3fc9bb22f%22+OR+%22uuid%3A031740b9-d472-4b35-8697-3efc830fd7fe%22+OR+%22uuid%3A031744d0-ea62-11e6-9964-005056825209%22+OR+%22uuid%3A03218800-bf0f-11e2-9592-5ef3fc9bb22f%22+OR+%22uuid%3A032185c1-ee27-11e2-9439-005056825209%22+OR+%22uuid%3A03218604-eb98-461c-8033-3dd1512169ef%22+OR+%22uuid%3A032187e2-f47b-47b5-a9ba-b0e00f12e8e9%22+OR+%22uuid%3A03174850-1edb-4aa1-acb1-4054fe2820f9%22+OR+%22uuid%3A03174a30-dac4-11e3-93a3-005056825209%22+OR+%22uuid%3A03174829-8301-413b-a982-2cfe1e6444bb%22+OR+%22uuid%3A03174c99-4e3b-464e-a65e-5d3d840f1693%22+OR+%22uuid%3A03218ea0-5522-11e8-ab92-5ef3fc9ae867%22+OR+%22uuid%3A03218c97-a41a-ffc3-fe18-4610acb2152a%22+OR+%22uuid%3A03218e20-b96e-11e5-b404-005056825209%22)&fl=pid+collection&wt=xml&rows=20"; +// String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("k7exists/1.xml"), "UTF-8"); +// List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); +// +// +// EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); +// +// // all updates in one check +// WebResource updateResource = EasyMock.createMock(WebResource.class); +// ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); +// WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); +// EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")).andReturn(updateResource).anyTimes(); +// EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); +// EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); +// +// //EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); +// //Capture captureSingleArgument = EasyMock.newCa +// Capture firstArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture secondArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(secondArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture thirdArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(thirdArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture fourthArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(fourthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture fifthArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(fifthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// +// EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); +// EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); +// EasyMock.expect(updateResponse.getEntityInputStream()).andDelegateTo(new MockClientResponse()).anyTimes(); +// +// // final commit +// WebResource commitResource = EasyMock.createMock(WebResource.class); +// WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); +// +// EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")).andReturn(commitResource).anyTimes(); +// EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); +// +// EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, +// commitResource, +// commitResourceBuilder); +// +// // check & fetch +// mocksFromSecondCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// process.setClient(client); +// +// // start whole process +// process.migrate(configurationFile); +// +// +// assertThat(Input.fromReader(new StringReader((String) firstArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("k7exists/k7batches/1_batch.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// +// } +// +// /** Copy and transform index K5->K7 - data are updated but in solr cloud*/ +// @Test +// public void testFullProcess_EXISTS_K7_TRANSFORM_SOLR_CLOUD() throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { +// +// InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("config.replicate.mlp_k7_solrcloud.xml"); +// String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); +// +// InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("k7exists/logfile_log"); +// String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); +// File logFile = File.createTempFile("junit","logfile"); +// FileUtils.write(logFile, logFileContent, "UTF-8"); +// +// String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); +// File configurationFile = File.createTempFile("junit","conf"); +// FileUtils.write(configurationFile, configurationContent, "UTF-8"); +// +// +// Client client = EasyMock.createMock(Client.class); +// ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class) +// .addMockedMethod("buildClient") +// .createMock(); +// +// // --- 20 indexed --- +// String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3A03173900-662a-11e3-9ea2-5ef3fc9ae867%22+OR+%22uuid%3A03173e90-9c18-11e3-a880-5ef3fc9ae867%22+OR+%22uuid%3A03218150-2e9e-11eb-bd18-005056827e51%22+OR+%22uuid%3A03218050-35fe-11e4-8e0d-005056827e51%22+OR+%22uuid%3A03174110-5186-11e3-8c6a-005056825209%22+OR+%22uuid%3A03174020-f2fa-11e6-bf7e-005056825209%22+OR+%22uuid%3A03173f70-5dc0-11e9-a9b4-5ef3fc9bb22f%22+OR+%22uuid%3A031740b9-d472-4b35-8697-3efc830fd7fe%22+OR+%22uuid%3A031744d0-ea62-11e6-9964-005056825209%22+OR+%22uuid%3A03218800-bf0f-11e2-9592-5ef3fc9bb22f%22+OR+%22uuid%3A032185c1-ee27-11e2-9439-005056825209%22+OR+%22uuid%3A03218604-eb98-461c-8033-3dd1512169ef%22+OR+%22uuid%3A032187e2-f47b-47b5-a9ba-b0e00f12e8e9%22+OR+%22uuid%3A03174850-1edb-4aa1-acb1-4054fe2820f9%22+OR+%22uuid%3A03174a30-dac4-11e3-93a3-005056825209%22+OR+%22uuid%3A03174829-8301-413b-a982-2cfe1e6444bb%22+OR+%22uuid%3A03174c99-4e3b-464e-a65e-5d3d840f1693%22+OR+%22uuid%3A03218ea0-5522-11e8-ab92-5ef3fc9ae867%22+OR+%22uuid%3A03218c97-a41a-ffc3-fe18-4610acb2152a%22+OR+%22uuid%3A03218e20-b96e-11e5-b404-005056825209%22)&fl=pid+collection+root.pid&wt=xml&rows=20"; +// String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("k7exists/1_solrcloud.xml"), "UTF-8"); +// List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); +// +// +// EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); +// +// // all updates in one check +// WebResource updateResource = EasyMock.createMock(WebResource.class); +// ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); +// WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); +// EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")).andReturn(updateResource).anyTimes(); +// EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); +// EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); +// +// //EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); +// //Capture captureSingleArgument = EasyMock.newCa +// Capture firstArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture secondArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(secondArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture thirdArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(thirdArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture fourthArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(fourthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture fifthArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(fifthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// +// EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); +// EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); +// EasyMock.expect(updateResponse.getEntityInputStream()).andDelegateTo(new MockClientResponse()).anyTimes(); +// +// // final commit +// WebResource commitResource = EasyMock.createMock(WebResource.class); +// WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); +// +// EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")).andReturn(commitResource).anyTimes(); +// EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); +// +// EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, +// commitResource, +// commitResourceBuilder); +// +// // check & fetch +// mocksFromSecondCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// process.setClient(client); +// +// // start whole process +// process.migrate(configurationFile); +// +// +// assertThat(Input.fromReader(new StringReader((String) firstArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("k7exists/k7batches/1_batch_solrcloud.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// } +// +// /** Copy and transform index K5->K7 - data are copied */ +// @Test +// public void testFullProcess_NOT_EXISTS_K7_TRANSFORM() throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { +// +// InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("config.replicate.mlp_k7.xml"); +// String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); +// +// InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("k7notexists/logfile_log"); +// String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); +// File logFile = File.createTempFile("junit","logfile"); +// FileUtils.write(logFile, logFileContent, "UTF-8"); +// +// String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); +// File configurationFile = File.createTempFile("junit","conf"); +// FileUtils.write(configurationFile, configurationContent, "UTF-8"); +// +// +// Client client = EasyMock.createMock(Client.class); +// ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class) +// .addMockedMethod("buildClient") +// .createMock(); +// +// // --- 20 indexed --- +// String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3Ae15e3a10-7774-11e7-89ee-5ef3fc9ae867%22+OR+%22uuid%3A420f4830-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Ae665ffa0-d6ad-11e7-8294-005056827e51%22+OR+%22uuid%3A88459bc0-cf86-11e7-a351-005056825209%22+OR+%22uuid%3A74bddf50-cf85-11e7-b059-5ef3fc9ae867%22+OR+%22uuid%3Ab7d82771-d69f-11e7-8294-005056827e51%22+OR+%22uuid%3A9f3f7bb0-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Af7217151-cf8a-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3A30bcc500-cf8f-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3Ae32eb050-d466-11e7-a047-005056825209%22+OR+%22uuid%3Ac39f12a0-d44a-11e7-a536-5ef3fc9ae867%22+OR+%22uuid%3Ab6632460-7641-11e4-9605-005056825209%22+OR+%22uuid%3A1e062920-d6be-11e7-9268-5ef3fc9ae867%22+OR+%22uuid%3Ab6c1a8f0-a730-11e2-b6da-005056827e52%22+OR+%22uuid%3A7c5e1e70-cc80-11e3-aec3-005056827e52%22+OR+%22uuid%3A8afd1150-1b3e-11e3-b62e-005056825209%22+OR+%22uuid%3A72356250-92c3-11e9-b601-005056825209%22+OR+%22uuid%3Aca058c00-929e-11e9-bc42-5ef3fc9ae867%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22)&fl=pid+collection&wt=xml&rows=20"; +// String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/1.xml"), "UTF-8"); +// List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); +// +// String secondReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3Ae15e3a10-7774-11e7-89ee-5ef3fc9ae867%22+OR+%22uuid%3A420f4830-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Ae665ffa0-d6ad-11e7-8294-005056827e51%22+OR+%22uuid%3A88459bc0-cf86-11e7-a351-005056825209%22+OR+%22uuid%3A74bddf50-cf85-11e7-b059-5ef3fc9ae867%22+OR+%22uuid%3Ab7d82771-d69f-11e7-8294-005056827e51%22+OR+%22uuid%3A9f3f7bb0-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Af7217151-cf8a-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3A30bcc500-cf8f-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3Ae32eb050-d466-11e7-a047-005056825209%22+OR+%22uuid%3Ac39f12a0-d44a-11e7-a536-5ef3fc9ae867%22+OR+%22uuid%3Ab6632460-7641-11e4-9605-005056825209%22+OR+%22uuid%3A1e062920-d6be-11e7-9268-5ef3fc9ae867%22+OR+%22uuid%3Ab6c1a8f0-a730-11e2-b6da-005056827e52%22+OR+%22uuid%3A7c5e1e70-cc80-11e3-aec3-005056827e52%22+OR+%22uuid%3A8afd1150-1b3e-11e3-b62e-005056825209%22+OR+%22uuid%3A72356250-92c3-11e9-b601-005056825209%22+OR+%22uuid%3Aca058c00-929e-11e9-bc42-5ef3fc9ae867%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; +// String secondRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/1_fetch.xml"), "UTF-8"); +// List mocksFromSecondFetchCall = webCallExpect(client, secondReqFetch, secondRespFetch); +// // ---------- +// +// // --- 3 indexed --- +// String thirdReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3A3b624880-96b5-11e2-9a08-005056827e52%22+OR+%22uuid%3A87d2a060-a692-11e2-8b87-005056827e51%22+OR+%22uuid%3A653cfcf0-3fac-11e7-b56f-005056827e52%22)&fl=pid+collection&wt=xml&rows=3"; +// String thirdResp = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/1.xml"), "UTF-8"); +// List mocksFromThirdCall = webCallExpect(client, thirdReq, thirdResp); +// +// String thirdReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A3b624880-96b5-11e2-9a08-005056827e52%22+OR+%22uuid%3A87d2a060-a692-11e2-8b87-005056827e51%22+OR+%22uuid%3A653cfcf0-3fac-11e7-b56f-005056827e52%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=3"; +// String thirdRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/2_fetch.xml"), "UTF-8"); +// List mocksFromThirdFetchCall = webCallExpect(client, thirdReqFetch, thirdRespFetch); +// // ---------- +// +// +// EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); +// +// // all updates in one check +// WebResource updateResource = EasyMock.createMock(WebResource.class); +// ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); +// WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); +// EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")).andReturn(updateResource).anyTimes(); +// EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); +// EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); +// +// //EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); +// //Capture captureSingleArgument = EasyMock.newCa +// Capture firstArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture secondArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(secondArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture thirdArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(thirdArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture fourthArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(fourthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture fifthArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(fifthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// +// EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); +// EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); +// EasyMock.expect(updateResponse.getEntityInputStream()).andDelegateTo(new MockClientResponse()).anyTimes(); +// +// // final commit +// WebResource commitResource = EasyMock.createMock(WebResource.class); +// WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); +// +// EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")).andReturn(commitResource).anyTimes(); +// EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); +// +// EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, +// commitResource, +// commitResourceBuilder); +// +// // check & fetch +// mocksFromSecondCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromSecondFetchCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// mocksFromThirdCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// mocksFromThirdFetchCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// process.setClient(client); +// +// // start whole process +// process.migrate(configurationFile); +// +// +// +// +// assertThat(Input.fromReader(new StringReader((String) firstArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("k7notexists/k7batches/1_batch.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// writeDebFile((String) secondArg.getValue()); +// +// assertThat(Input.fromReader(new StringReader((String) secondArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("k7notexists/k7batches/2_batch.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// } +// +// private void writeDebFile(String str) { +// try { +// File f = new File("test.batch"); +// f.createNewFile(); +// IOUtils.write(str, new FileOutputStream(f)); +// +// } catch (IOException e) { +// // TODO Auto-generated catch block +// e.printStackTrace(); +// } +// } +// +// @Test +// public void testFullProcess_NOT_EXISTS_K7_TRANSFORM_SOLR_CLOUD() throws IOException, IllegalAccessException, InstantiationException, ClassNotFoundException, SAXException, ParserConfigurationException, NoSuchMethodException, MigrateSolrIndexException, URISyntaxException { +// +// InputStream configurationStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("config.replicate.mlp_k7_solrcloud.xml"); +// String _configurationContent = IOUtils.toString(configurationStream, "UTF-8"); +// +// InputStream logFileStream = ParallelProcessImplTest_REPLICATE.class.getResourceAsStream("k7notexists/logfile_log"); +// String logFileContent = IOUtils.toString(logFileStream, "UTF-8"); +// File logFile = File.createTempFile("junit","logfile"); +// FileUtils.write(logFile, logFileContent, "UTF-8"); +// +// String configurationContent = String.format(_configurationContent, logFile.toURI().toURL().toString()); +// File configurationFile = File.createTempFile("junit","conf"); +// FileUtils.write(configurationFile, configurationContent, "UTF-8"); +// +// +// Client client = EasyMock.createMock(Client.class); +// ParallelProcessImpl process = createMockBuilder(ParallelProcessImpl.class) +// .addMockedMethod("buildClient") +// .createMock(); +// +// // --- 20 indexed --- +// String secondReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3Ae15e3a10-7774-11e7-89ee-5ef3fc9ae867%22+OR+%22uuid%3A420f4830-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Ae665ffa0-d6ad-11e7-8294-005056827e51%22+OR+%22uuid%3A88459bc0-cf86-11e7-a351-005056825209%22+OR+%22uuid%3A74bddf50-cf85-11e7-b059-5ef3fc9ae867%22+OR+%22uuid%3Ab7d82771-d69f-11e7-8294-005056827e51%22+OR+%22uuid%3A9f3f7bb0-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Af7217151-cf8a-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3A30bcc500-cf8f-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3Ae32eb050-d466-11e7-a047-005056825209%22+OR+%22uuid%3Ac39f12a0-d44a-11e7-a536-5ef3fc9ae867%22+OR+%22uuid%3Ab6632460-7641-11e4-9605-005056825209%22+OR+%22uuid%3A1e062920-d6be-11e7-9268-5ef3fc9ae867%22+OR+%22uuid%3Ab6c1a8f0-a730-11e2-b6da-005056827e52%22+OR+%22uuid%3A7c5e1e70-cc80-11e3-aec3-005056827e52%22+OR+%22uuid%3A8afd1150-1b3e-11e3-b62e-005056825209%22+OR+%22uuid%3A72356250-92c3-11e9-b601-005056825209%22+OR+%22uuid%3Aca058c00-929e-11e9-bc42-5ef3fc9ae867%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22)&fl=pid+collection+root.pid&wt=xml&rows=20"; +// String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/1.xml"), "UTF-8"); +// List mocksFromSecondCall = webCallExpect(client, secondReq, secondResp); +// +// String secondReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3Ae15e3a10-7774-11e7-89ee-5ef3fc9ae867%22+OR+%22uuid%3A420f4830-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Ae665ffa0-d6ad-11e7-8294-005056827e51%22+OR+%22uuid%3A88459bc0-cf86-11e7-a351-005056825209%22+OR+%22uuid%3A74bddf50-cf85-11e7-b059-5ef3fc9ae867%22+OR+%22uuid%3Ab7d82771-d69f-11e7-8294-005056827e51%22+OR+%22uuid%3A9f3f7bb0-d6b2-11e7-8558-005056825209%22+OR+%22uuid%3Af7217151-cf8a-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3A30bcc500-cf8f-11e7-80e7-5ef3fc9bb22f%22+OR+%22uuid%3Ae32eb050-d466-11e7-a047-005056825209%22+OR+%22uuid%3Ac39f12a0-d44a-11e7-a536-5ef3fc9ae867%22+OR+%22uuid%3Ab6632460-7641-11e4-9605-005056825209%22+OR+%22uuid%3A1e062920-d6be-11e7-9268-5ef3fc9ae867%22+OR+%22uuid%3Ab6c1a8f0-a730-11e2-b6da-005056827e52%22+OR+%22uuid%3A7c5e1e70-cc80-11e3-aec3-005056827e52%22+OR+%22uuid%3A8afd1150-1b3e-11e3-b62e-005056825209%22+OR+%22uuid%3A72356250-92c3-11e9-b601-005056825209%22+OR+%22uuid%3Aca058c00-929e-11e9-bc42-5ef3fc9ae867%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22+OR+%22uuid%3A69eb8720-932f-11e9-b601-005056825209%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=20"; +// String secondRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/1_fetch.xml"), "UTF-8"); +// List mocksFromSecondFetchCall = webCallExpect(client, secondReqFetch, secondRespFetch); +// // ---------- +// +// // --- 3 indexed --- +// String thirdReq = "http://10.19.6.10:8983/solr/k7_5/select/?q=pid:(%22uuid%3A3b624880-96b5-11e2-9a08-005056827e52%22+OR+%22uuid%3A87d2a060-a692-11e2-8b87-005056827e51%22+OR+%22uuid%3A653cfcf0-3fac-11e7-b56f-005056827e52%22)&fl=pid+collection+root.pid&wt=xml&rows=3"; +// String thirdResp = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/1.xml"), "UTF-8"); +// List mocksFromThirdCall = webCallExpect(client, thirdReq, thirdResp); +// +// String thirdReqFetch = "http://kramerius4.mlp.cz/search/api/v5.0/search/?q=PID:(%22uuid%3A3b624880-96b5-11e2-9a08-005056827e52%22+OR+%22uuid%3A87d2a060-a692-11e2-8b87-005056827e51%22+OR+%22uuid%3A653cfcf0-3fac-11e7-b56f-005056827e52%22)&fl=PID+timestamp+fedora.model+document_type+handle+status+created_date+modified_date+parent_model+parent_pid+parent_pid+parent_title+root_model+root_pid+root_title+text_ocr+pages_count+datum_str+datum+rok+datum_begin+datum_end+datum_page+issn+mdt+ddt+dostupnost+keywords+geographic_names+collection+sec+model_path+pid_path+rels_ext_index+level+dc.title+title_sort+title_sort+dc.creator+dc.identifier+language+dc.description+details+facet_title+browse_title+browse_autor+img_full_mime+viewable+virtual+location+range+mods.shelfLocator+mods.physicalLocation+text+dnnt+dnnt-labels&wt=xml&rows=3"; +// String thirdRespFetch = IOUtils.toString(this.getClass().getResourceAsStream("k7notexists/2_fetch.xml"), "UTF-8"); +// List mocksFromThirdFetchCall = webCallExpect(client, thirdReqFetch, thirdRespFetch); +// // ---------- +// +// +// EasyMock.expect(process.buildClient()).andReturn(client).anyTimes(); +// +// // all updates in one check +// WebResource updateResource = EasyMock.createMock(WebResource.class); +// ClientResponse updateResponse = EasyMock.createMock(ClientResponse.class); +// WebResource.Builder updateResponseBuilder = EasyMock.createMock(WebResource.Builder.class); +// EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update")).andReturn(updateResource).anyTimes(); +// EasyMock.expect(updateResource.accept(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); +// EasyMock.expect(updateResponseBuilder.type(MediaType.TEXT_XML)).andReturn(updateResponseBuilder).anyTimes(); +// +// //EasyMock.expect(updateResponseBuilder.entity(EasyMock.anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).anyTimes(); +// //Capture captureSingleArgument = EasyMock.newCa +// Capture firstArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(firstArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture secondArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(secondArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture thirdArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(thirdArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture fourthArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(fourthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// Capture fifthArg = newCapture(); +// EasyMock.expect(updateResponseBuilder.entity(capture(fifthArg), EasyMock.eq(MediaType.TEXT_XML))).andReturn(updateResponseBuilder).once(); +// +// +// EasyMock.expect(updateResponseBuilder.post(ClientResponse.class)).andReturn(updateResponse).anyTimes(); +// EasyMock.expect(updateResponse.getStatus()).andReturn(ClientResponse.Status.OK.getStatusCode()).anyTimes(); +// EasyMock.expect(updateResponse.getEntityInputStream()).andDelegateTo(new MockClientResponse()).anyTimes(); +// +// // final commit +// WebResource commitResource = EasyMock.createMock(WebResource.class); +// WebResource.Builder commitResourceBuilder = EasyMock.createMock(WebResource.Builder.class); +// +// EasyMock.expect(client.resource("http://192.168.10.109:18984/solr-test/kramerius-cdk-test/update?commit=true")).andReturn(commitResource).anyTimes(); +// EasyMock.expect(commitResource.accept(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.type(MediaType.TEXT_XML)).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.entity(anyObject(String.class), EasyMock.eq(MediaType.TEXT_XML))).andReturn(commitResourceBuilder).anyTimes(); +// EasyMock.expect(commitResourceBuilder.post(String.class)).andReturn("").anyTimes(); +// +// EasyMock.replay(client, process, updateResource, updateResponse, updateResponseBuilder, +// commitResource, +// commitResourceBuilder); +// +// // check & fetch +// mocksFromSecondCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// mocksFromSecondFetchCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// mocksFromThirdCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// mocksFromThirdFetchCall.stream().forEach(obj-> { +// EasyMock.replay(obj); +// }); +// +// process.setClient(client); +// +// // start whole process +// process.migrate(configurationFile); +// +// +// assertThat(Input.fromReader(new StringReader((String) firstArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("k7notexists/k7batches/1_batch_solrcloud.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// +// writeDebFile((String) secondArg.getValue()); +// +// +// assertThat(Input.fromReader(new StringReader((String) secondArg.getValue()))) +// .and(Input.fromStream(this.getClass().getResourceAsStream("k7notexists/k7batches/2_batch_solrcloud.xml"))) +// .ignoreComments() +// .ignoreWhitespace() +// .areSimilar(); +// +// } static class MockClientResponse extends ClientResponse { diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java index fdc96ef6fc..d571089348 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -1,203 +1,140 @@ package cz.incad.kramerius; -import static cz.incad.kramerius.services.iterators.utils.IterationUtils.pidsToIterationItem; -import static cz.incad.kramerius.services.utils.SolrUtils.findAllPids; - -import java.io.File; import java.io.IOException; -import java.io.InputStream; -import java.io.StringWriter; import java.io.UnsupportedEncodingException; -import java.net.URL; -import java.net.URLEncoder; -import java.util.ArrayList; -import java.util.LinkedHashSet; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TimeZone; -import java.util.logging.Level; import java.util.logging.Logger; +import javax.ws.rs.Path; import javax.ws.rs.core.MediaType; import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.TransformerException; -import org.antlr.stringtemplate.StringTemplate; -import org.antlr.stringtemplate.language.DefaultTemplateLexer; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.tuple.Pair; import org.json.JSONArray; import org.json.JSONObject; -import org.w3c.dom.Document; -import org.w3c.dom.Element; import org.xml.sax.SAXException; import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.json.JSONConfiguration; -import cz.incad.kramerius.services.KubernetesProcessImpl; -import cz.incad.kramerius.services.ParallelProcessImpl; +import cz.incad.kramerius.service.MigrateSolrIndexException; import cz.incad.kramerius.services.utils.kubernetes.KubernetesEnvSupport; -import cz.incad.kramerius.utils.XMLUtils; - -import static cz.incad.kramerius.services.iterators.solr.SolrCursorIterator.*; - +import cz.incad.kramerius.utils.ReharvestUtils; public class KubernetesReharvestProcess { + public static final String ONLY_SHOW_CONFIGURATION = "ONLY_SHOW_CONFIGURATION"; + public static final Logger LOGGER = Logger.getLogger(KubernetesReharvestProcess.class.getName()); - - + protected static Client buildClient() { - //Client client = Client.create(); + // Client client = Client.create(); ClientConfig cc = new DefaultClientConfig(); - cc.getProperties().put( - ClientConfig.PROPERTY_FOLLOW_REDIRECTS, true); + cc.getProperties().put(ClientConfig.PROPERTY_FOLLOW_REDIRECTS, true); cc.getFeatures().put(JSONConfiguration.FEATURE_POJO_MAPPING, Boolean.TRUE); return Client.create(cc); - } + } + + public static void main(String[] args) throws ParserConfigurationException, IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchMethodException, MigrateSolrIndexException, IOException, SAXException { - - public static void main(String[] args) throws UnsupportedEncodingException, ParserConfigurationException { - TimeZone.setDefault(TimeZone.getTimeZone("Europe/Prague")); Map env = System.getenv(); Map iterationMap = KubernetesEnvSupport.iterationMap(env); - Map proxyMap = KubernetesEnvSupport.reharvestMap(env); - if (proxyMap.containsKey("url")) { - Client client= buildClient(); - String wurl = proxyMap.get("url"); + Map reharvestMap = KubernetesEnvSupport.reharvestMap(env); + Map destinationMap = KubernetesEnvSupport.destinationMap(env); + Map proxyMap = KubernetesEnvSupport.proxyMap(env); + boolean onlyShowConfiguration = env.containsKey(ONLY_SHOW_CONFIGURATION); + + + if (reharvestMap.containsKey("url") && proxyMap.containsKey("url")) { + Client client = buildClient(); + String wurl = reharvestMap.get("url"); if (!wurl.endsWith("/")) { - wurl = wurl +"/"; + wurl = wurl + "/"; } - WebResource r = client.resource(wurl + "top"); - ClientResponse clientResponse = r.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); - if (clientResponse.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + // Top item + WebResource topWebResource = client.resource(wurl + "top"); + ClientResponse topItemFrom = topWebResource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + if (topItemFrom.getStatus() == ClientResponse.Status.OK.getStatusCode()) { - String t = clientResponse.getEntity(String.class); + String t = topItemFrom.getEntity(String.class); JSONObject itemObject = new JSONObject(t); String id = itemObject.getString("id"); - - LinkedHashSet allPids = new LinkedHashSet<>(); JSONArray pids = itemObject.getJSONArray("pids"); - for (int i = 0; i < pids.length(); i++) { - String pid = pids.getString(i); - allPids.add(pid); - String iterationUrl = iterationMap.get("url"); - String masterQuery = "*:*"; - String filterQuery = "root.pid:\""+pid+"\""; - try { - String cursorMark = null; - String queryCursorMark = null; - do { - Element element = pidsCursorQuery(client, iterationUrl, masterQuery, cursorMark, 100, filterQuery, "select", "compositeId+pid", "compositeId asc", "", ""); - cursorMark = findCursorMark(element); - queryCursorMark = findQueryCursorMark(element); - allPids.addAll(findAllPids(element)); - } while((cursorMark != null && queryCursorMark != null) && !cursorMark.equals(queryCursorMark)); - } catch (ParserConfigurationException | SAXException | IOException e) { - throw new RuntimeException(e); - } - } - - //System.out.println("Number of pids "+allPids.size()); - List allPidsList = new ArrayList<>(); - allPidsList.addAll(allPids); - allPidsList.sort(String::compareTo); - - int batchSize = 42; - int batches = allPidsList.size() / batchSize; - if (allPidsList.size() % batchSize > 0) { - batches = batches + 1; - } - - for (int i = 0; i < batches; i++) { - int min = i* batchSize; - int max = Math.min((i+1)*batchSize, allPidsList.size()); + // find all pids + List> allPidsList = ReharvestUtils.findAllPidsByGivenRootPid(iterationMap, client, pids); + // delete all pids + ReharvestUtils.deleteAllGivenPids(client, destinationMap, allPidsList, onlyShowConfiguration); + + // List acronyms = new ArrayList(); + // https://api.val.ceskadigitalniknihovna.cz/search/api/admin/v7.0/connected/ + String proxyURl = proxyMap.get("url"); + if (proxyURl != null) { + Map configurations = libraryConfigurations(client, proxyURl); - Document deleteBatch = XMLUtils.crateDocument("delete"); - List batchPids = allPidsList.subList(min, max); - batchPids.forEach(pid-> { - Element idElm = deleteBatch.createElement("id"); - idElm.setTextContent(pid.trim()); - deleteBatch.getDocumentElement().appendChild(idElm); - }) ; + for (int i = 0; i < pids.length(); i++) { + String p = pids.getString(i); + // reharvesting + ReharvestUtils.reharvestPIDFromGivenCollections(p, configurations, ""+onlyShowConfiguration, destinationMap); + } - try { - StringWriter writer = new StringWriter(); - XMLUtils.print(deleteBatch, writer); - System.out.println(writer.toString()); - } catch (TransformerException e) { - e.printStackTrace(); + if (!onlyShowConfiguration) { + // @Path("{id}/state") + WebResource deleteWebResource = client.resource(wurl + id+"/state?state=closed"); + ClientResponse deleteResponse = deleteWebResource.accept(MediaType.APPLICATION_JSON).put(ClientResponse.class); + if (deleteResponse.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + LOGGER.info("Reharvest item finished"); + } } + } else { + LOGGER.severe("No proxy configuration"); } - - } else if (clientResponse.getStatus() == ClientResponse.Status.NOT_FOUND.getStatusCode()) { - LOGGER.log(Level.SEVERE,"No item for reharvest"); - } else { - throw new UniformInterfaceException(clientResponse); + } else if (topItemFrom.getStatus() == ClientResponse.Status.NOT_FOUND.getStatusCode()) { + LOGGER.info("No item to harvest"); } + } else { + LOGGER.severe("No proxy or reharvest configuration"); + } + } - -// Map iteration = KubernetesEnvSupport.iterationMap(env); -// Map check = KubernetesEnvSupport.checkMap(env); -// Map destination = KubernetesEnvSupport.destinationMap(env); -// Map timestamps = KubernetesEnvSupport.timestampMap(env, destination); -// -// StringTemplate template = new StringTemplate( -// IOUtils.toString(stream, "UTF-8"), DefaultTemplateLexer.class); - + public static Map libraryConfigurations(Client client, String proxyURl) { + Map configurations = new HashMap<>(); + WebResource proxyWebResource = client.resource(proxyURl); + ClientResponse allConnectedItems = proxyWebResource.accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + if (allConnectedItems.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + String responseAllConnected =allConnectedItems.getEntity(String.class); + JSONObject responseAllConnectedObject = new JSONObject(responseAllConnected); + for (Object key : responseAllConnectedObject.keySet()) { + JSONObject lib = responseAllConnectedObject.getJSONObject(key.toString()); + if (lib.has("status")) { + String configURl = proxyURl; + if (!configURl.endsWith("/")) { + configURl += "/"; + } + configURl += key + "/config"; + + WebResource configResource = client.resource(configURl); + ClientResponse configReourceStatus = configResource.accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + if (configReourceStatus.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + configurations.put(key.toString(), + new JSONObject(configReourceStatus.getEntity(String.class))); + } + } + } } - - - // -// if (env.containsKey(KubernetesEnvSupport.CONFIG_SOURCE) || args.length > 0) { -// String configSource = env.containsKey(KubernetesEnvSupport.CONFIG_SOURCE) ? env.get(KubernetesEnvSupport.CONFIG_SOURCE) : args[0]; -// InputStream stream = KubernetesProcessImpl.class.getResourceAsStream(configSource); -// if (configSource.trim().startsWith("file:///")) { -// URL fileUrl = new URL(configSource); -// stream = fileUrl.openStream(); -// } -// if (stream != null) { -// -// Map iteration = KubernetesEnvSupport.iterationMap(env); -// Map check = KubernetesEnvSupport.checkMap(env); -// Map destination = KubernetesEnvSupport.destinationMap(env); -// Map timestamps = KubernetesEnvSupport.timestampMap(env, destination); -// -// StringTemplate template = new StringTemplate( -// IOUtils.toString(stream, "UTF-8"), DefaultTemplateLexer.class); -// -// template.setAttribute("iteration", iteration); -// template.setAttribute("check", check); -// template.setAttribute("destination", destination); -// template.setAttribute("timestamp", timestamps); -// -// String configuration = template.toString(); -// LOGGER.info("Loading configuration "+configuration); -// -// File tmpFile = File.createTempFile("temp", "file"); -// FileUtils.write(tmpFile, configuration, "UTF-8"); -// -// -// if (!env.containsKey(ONLY_SHOW_CONFIGURATION)) { -// ParallelProcessImpl migr = new ParallelProcessImpl(); -// migr.migrate(tmpFile); -// } -// -// } else { -// LOGGER.severe(String.format("Cannot find resource %s", configSource)); -// } -// } + return configurations; } - } diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java deleted file mode 100644 index a51ca1fc8d..0000000000 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/ReharvestPids.java +++ /dev/null @@ -1,359 +0,0 @@ -package cz.incad.kramerius; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.StringWriter; -import java.net.URLEncoder; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.logging.Level; -import java.util.logging.Logger; -import java.util.stream.Collectors; - -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.TransformerException; - -import org.antlr.stringtemplate.StringTemplate; -import org.antlr.stringtemplate.language.DefaultTemplateLexer; -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.IOUtils; -import org.apache.commons.lang3.tuple.Pair; -import org.apache.solr.client.solrj.SolrServerException; -import org.w3c.dom.DOMException; -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.xml.sax.SAXException; - -import com.google.common.io.Files; -import com.google.inject.Guice; -import com.google.inject.Injector; -import com.google.inject.Key; -import com.hazelcast.config.KubernetesConfig; - -import cz.incad.kramerius.fedora.RepoModule; -import cz.incad.kramerius.fedora.om.RepositoryException; -import cz.incad.kramerius.impl.SolrAccessImplNewIndex; -import cz.incad.kramerius.resourceindex.ResourceIndexException; -import cz.incad.kramerius.resourceindex.ResourceIndexModule; -import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; -import cz.incad.kramerius.rest.apiNew.client.v60.libs.properties.DefaultPropertiesInstances; -import cz.incad.kramerius.service.MigrateSolrIndexException; -import cz.incad.kramerius.solr.SolrModule; -import cz.incad.kramerius.statistics.NullStatisticsModule; -import cz.incad.kramerius.utils.StringUtils; -import cz.incad.kramerius.utils.XMLUtils; -import cz.incad.kramerius.utils.conf.KConfiguration; -//import cz.incad.kramerius.utils.solr.SolrUtils; -import cz.kramerius.searchIndex.indexer.SolrConfig; -import cz.kramerius.searchIndex.indexer.SolrIndexAccess; -import cz.incad.kramerius.services.KubernetesProcessImpl; -import cz.incad.kramerius.services.ParallelProcessImpl; -import cz.incad.kramerius.services.utils.ResultsUtils; -import cz.incad.kramerius.services.utils.SolrUtils; - - -public class ReharvestPids { - - public static final Logger LOGGER = Logger.getLogger(ReharvestPids.class.getName()); - - - public static void main(String[] args) throws IOException, SolrServerException, RepositoryException, ResourceIndexException, TransformerException, ParserConfigurationException, MigrateSolrIndexException { - if (args.length < 2) { - throw new RuntimeException("Not enough arguments."); - } - LOGGER.info("Parameters "+Arrays.asList(args)); - int argsIndex = 0; -// //params from lp.st - String authToken = args[argsIndex++]; - String target = args[argsIndex++]; //auth token always second, but still suboptimal solution, best would be if it was outside the scope of this as if ProcessHelper.scheduleProcess() similarly to changing name (ProcessStarter) - String onlyShowConfiguration = null; - if (args.length>=3) { - onlyShowConfiguration = args[argsIndex++]; - } - - List extractPids = extractPids(target); - for (String pid : extractPids) { reharvestPID(pid, onlyShowConfiguration); } - } - - private static void reharvestPID(String pid, String onlyShowConfiguration) - throws IOException, TransformerException, MigrateSolrIndexException, ParserConfigurationException { - Injector injector = Guice.createInjector(new SolrModule(), new ResourceIndexModule(), new RepoModule(), new NullStatisticsModule(), new ResourceIndexModule()); - - SolrAccess searchIndex = injector.getInstance(Key.get(SolrAccessImplNewIndex.class)); //FIXME: hardcoded implementation - Document document = searchIndex.getSolrDataByPid(pid); - - if (document != null) { - // nalezen v solru - int nf = numFound(document); - if (nf > 0) { - - String msg = String.format("Reharvesting pid %s ", pid); - LOGGER.info(msg); - - String leader = leader(document); - List collections = collections(document); - - ParallelProcessImpl parallelProcess = new ParallelProcessImpl(); - Document deleteByQuery = deleteRootPid(searchIndex, pid); - if (isOnlyShowConfiguration(onlyShowConfiguration)) { - StringWriter writer = new StringWriter(); - XMLUtils.print(deleteByQuery, writer); - LOGGER.info("Delete by query "+writer.toString()); - } else { - StringWriter writer = new StringWriter(); - XMLUtils.print(deleteByQuery, writer); - LOGGER.info("Delete by query; only show query and number off docs :"+writer.toString()); - String s = SolrUtils.sendToDest(getDestinationUpdateUrl(), parallelProcess.getClient(), deleteByQuery); - } - - collections.remove(leader); - collections.add(0, leader); - reharvestPIDFromGivenCollections(pid, collections, onlyShowConfiguration); - } else { - DefaultPropertiesInstances props = new DefaultPropertiesInstances(); - List enabledInstances = props.enabledInstances(); - reharvestPIDFromGivenCollections(pid, enabledInstances.stream().map(OneInstance::getName).collect(Collectors.toList()), onlyShowConfiguration); - } - } - } - - - private static boolean isOnlyShowConfiguration(String onlyShowConfiguration) { - if (onlyShowConfiguration != null && ("onlyshowconfiguration".equals(onlyShowConfiguration.toLowerCase()) || "true".equals(onlyShowConfiguration))) { - return true; - } - return false; - } - - private static void reharvestPIDFromGivenCollections(String pid, List collections, String onlyShowConfiguration) { - List harvestFiles = new ArrayList<>(); - for (String ac : collections) { - try { - String api = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + ac + ".api"); - if (api == null) { - LOGGER.warning(String.format("Skipping instance %s", ac)); - continue; - } - - Pair iterationUrl = iterationUrl(ac); - Map iteration = new HashMap<>(); - iteration.put("url", iterationUrl.getKey()); - iteration.put("dl", ac); - iteration.put("fquery", fq(api, pid)); - - - if (iterationUrl.getRight()) { - String username = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + ac + ".username"); - String password = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + ac + ".pswd"); - - iteration.put("user", username); - iteration.put("pass", password); - } - - // destination - Map destination = new HashMap<>(); - destination.put("url", KConfiguration.getInstance().getSolrSearchHost()); - - boolean publicEndpoint = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + ac + ".public") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + ac + ".public") : false; - - String configuration = renderTemplate(publicEndpoint, api, iteration, destination); - File tmpFile = File.createTempFile(String.format("%s", ac), "reharvest"); - - Files.write(configuration.getBytes("UTF-8"), tmpFile); - harvestFiles.add(tmpFile); - - } catch (IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } - } - - for (File harvestFile : harvestFiles) { - try { - if (isOnlyShowConfiguration(onlyShowConfiguration)) { - String config = org.apache.commons.io.IOUtils.toString(new FileInputStream(harvestFile), "UTF-8"); - LOGGER.info(String.format("Configuration %s" ,config)); - } else { - // safra ?? - ParallelProcessImpl reharvest = new ParallelProcessImpl(); - String config = org.apache.commons.io.IOUtils.toString(new FileInputStream(harvestFile), "UTF-8"); - LOGGER.info(String.format("Configuration %s" ,config)); - reharvest.migrate(harvestFile); - } - - } catch (IOException | MigrateSolrIndexException | IllegalAccessException | InstantiationException | ClassNotFoundException | NoSuchMethodException | ParserConfigurationException | SAXException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } - } - } - - private static String fq(String api, String pid) { - switch(api) { - case "v7": return "root.pid:\""+pid+"\""; - case "v5": return "root_pid:\""+pid+"\""; - } - return "root_pid:\""+pid+"\""; - } - - private static String renderTemplate(boolean pbl, String api, Map iteration, Map destination) throws IOException { - String templatePath = String.format("reharvest_%s.xml", api); - if (pbl) { - templatePath = String.format("reharvest_public_%s.xml", api); - } - - InputStream stream = ReharvestPids.class.getResourceAsStream(templatePath); - - StringTemplate template = new StringTemplate( - org.apache.commons.io.IOUtils.toString(stream, "UTF-8"), DefaultTemplateLexer.class); - - template.setAttribute("iteration", iteration); - template.setAttribute("check", new HashMap<>()); - template.setAttribute("destination", destination); - template.setAttribute("timestamp", new HashMap<>()); - - String configuration = template.toString(); - return configuration; - } - - public static String getDestinationUpdateUrl() { - String searchHost = KConfiguration.getInstance().getSolrSearchHost(); - return searchHost +(searchHost.endsWith("/") ? "" : "/")+"update?commit=true"; - } - - public static Pair iterationUrl(String acronym) { - String baseurl = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + acronym + ".baseurl"); - String api = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + acronym + ".api"); - boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + acronym + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + acronym + ".licenses") : false; - String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + acronym + ".forwardurl"); - if (channelAccess) { - if (StringUtils.isAnyString(api) && api.toLowerCase().equals("v5")) { - String retval = channel + (channel.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/sync/solr"; - return Pair.of(retval, false); - } else { - String retval = channel + (channel.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/sync/solr"; - return Pair.of(retval, false); - } - } else { - boolean publicEndpoint = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + acronym + ".public") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + acronym + ".public") : false; - if (publicEndpoint) { - String retval = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0"; - return Pair.of(retval, false); - } else { - String retval = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v4.6/cdk/solr"; - return Pair.of(retval, true); - - } - } - } - - - private static Document deleteRootPid(SolrAccess solrAccess, String pid) throws ParserConfigurationException, DOMException, IOException { - Document deleteByQuery = XMLUtils.crateDocument("delete"); - Element queryElement = deleteByQuery.createElement("query"); - queryElement.setTextContent(query(solrAccess, pid)); - deleteByQuery.getDocumentElement().appendChild(queryElement); - return deleteByQuery; - } - - private static String query(SolrAccess sAccess, String pid) throws IOException { - String query = String.format("root.pid:\"%s\"",pid); - if (query.startsWith("root.pid:\"uuid")) { - - int threshold = KConfiguration.getInstance().getConfiguration().getInt("cdk.reharvest.items.threshold",30000); - Document doc = sAccess.requestWithSelectReturningXml("q="+URLEncoder.encode( query, "UTF-8")); - int numFound = numFound(doc); - LOGGER.info(String.format( "Number of found documents %d", numFound)); - if (numFound > threshold) { - throw new IllegalStateException("Too many items to reharvest"); - } - return query; - } else throw new IllegalStateException("Pid must start with uuid !"); - } - -// private static Document deleteOwnPidPathSubtree(String pidPath) throws ParserConfigurationException { -// Document deleteByQuery = XMLUtils.crateDocument("delete"); -// Element queryElement = deleteByQuery.createElement("query"); -// queryElement.setTextContent(String.format("own_pid_path:%s/*", pidPath.replace(":", "\\:"))); -// deleteByQuery.getDocumentElement().appendChild(queryElement); -// return deleteByQuery; -// } - - - private static List collections(Document document) { - Element cdkCollections = XMLUtils.findElement(document.getDocumentElement(), new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - String name = element.getAttribute("name"); - return name != null && name.equals("cdk.collection"); - } - }); - return cdkCollections != null ? XMLUtils.getElements(cdkCollections).stream().map(Element::getTextContent).map(String::trim).collect(Collectors.toList()) : new ArrayList<>(); - } - - private static String leader(Document document) { - Element cdkLeader = XMLUtils.findElement(document.getDocumentElement(), new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - String name = element.getAttribute("name"); - return name != null && name.equals("cdk.leader"); - } - }); - return cdkLeader != null ? cdkLeader.getTextContent().trim() : null; - } - - public static int numFound(Document document) { - Element result = XMLUtils.findElement(document.getDocumentElement(), new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - String nodeName = element.getNodeName(); - return nodeName.equals("result"); - } - }); - - String nf = result.getAttribute("numFound"); - return Integer.parseInt(nf); - } - - -// private static String ownPidPath(Document document) { -// Element cdkLeader = XMLUtils.findElement(document.getDocumentElement(), new XMLUtils.ElementsFilter() { -// @Override -// public boolean acceptElement(Element element) { -// String name = element.getAttribute("name"); -// return name != null && name.equals("own_pid_path"); -// } -// }); -// return cdkLeader != null ? cdkLeader.getTextContent().trim() : null; -// } - - private static List extractPids(String target) { - if (target.startsWith("pid:")) { - String pid = target.substring("pid:".length()); - List result = new ArrayList<>(); - result.add(pid); - return result; - } else if (target.startsWith("pidlist:")) { - List pids = Arrays.stream(target.substring("pidlist:".length()).split(";")).map(String::trim).filter(s -> !s.isEmpty()).collect(Collectors.toList()); - return pids; - } else if (target.startsWith("pidlist_file:")) { - String filePath = target.substring("pidlist_file:".length()); - File file = new File(filePath); - if (file.exists()) { - try { - return IOUtils.readLines(new FileInputStream(file), Charset.forName("UTF-8")); - } catch (IOException e) { - throw new RuntimeException("IOException " + e.getMessage()); - } - } else { - throw new RuntimeException("file " + file.getAbsolutePath() + " doesnt exist "); - } - } else { - throw new RuntimeException("invalid target " + target); - } - } -} diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java new file mode 100644 index 0000000000..c931224a68 --- /dev/null +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -0,0 +1,233 @@ +package cz.incad.kramerius.utils; + +import static cz.incad.kramerius.services.iterators.solr.SolrCursorIterator.findCursorMark; +import static cz.incad.kramerius.services.iterators.solr.SolrCursorIterator.findQueryCursorMark; +import static cz.incad.kramerius.services.iterators.solr.SolrCursorIterator.pidsCursorQuery; +import static cz.incad.kramerius.services.utils.SolrUtils.findAllPids; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.StringWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.logging.Level; +import java.util.logging.Logger; +import java.util.stream.Collectors; + +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; + +import org.antlr.stringtemplate.StringTemplate; +import org.antlr.stringtemplate.language.DefaultTemplateLexer; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.json.JSONArray; +import org.json.JSONObject; +import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import com.google.common.io.Files; +import com.sun.jersey.api.client.Client; + +import cz.incad.kramerius.KubernetesReharvestProcess; +import cz.incad.kramerius.service.MigrateSolrIndexException; +import cz.incad.kramerius.services.ParallelProcessImpl; +import cz.incad.kramerius.services.utils.SolrUtils; + +public class ReharvestUtils { + + public static final Logger LOGGER = Logger.getLogger(ReharvestUtils.class.getName()); + + private ReharvestUtils() {} + + /** Delete all given pids */ + public static void deleteAllGivenPids(Client client,Map destinationMap, List> allPidsList, boolean onlyShowConfig) + throws ParserConfigurationException { + int batchSize = 42; + int batches = allPidsList.size() / batchSize; + if (allPidsList.size() % batchSize > 0) { + batches = batches + 1; + } + for (int i = 0; i < batches; i++) { + int min = i * batchSize; + int max = Math.min((i + 1) * batchSize, allPidsList.size()); + Document deleteBatch = XMLUtils.crateDocument("delete"); + List> batchPids = allPidsList.subList(min, max); + + batchPids.forEach(pid -> { + Element idElm = deleteBatch.createElement("id"); + idElm.setTextContent(pid.getRight().trim()); + deleteBatch.getDocumentElement().appendChild(idElm); + }); + + try { + + if (!onlyShowConfig) { + String destinationUrl = destinationMap.get("url")+"/update?commit=true"; + String s = SolrUtils.sendToDest(destinationUrl, client, deleteBatch); + } else { + StringWriter writer = new StringWriter(); + XMLUtils.print(deleteBatch, writer); + LOGGER.info(writer.toString()); + } + + } catch (Exception e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } + } + } + + /** Find all pids by given root.pid */ + public static List> findAllPidsByGivenRootPid(Map iterationMap, Client client, JSONArray pids) { + LinkedHashSet> allPids = new LinkedHashSet<>(); + for (int i = 0; i < pids.length(); i++) { + String pid = pids.getString(i); + //allPids.add(pid); + String iterationUrl = iterationMap.get("url"); + String masterQuery = "*:*"; + String filterQuery = "root.pid:\"" + pid + "\""; + try { + String cursorMark = null; + String queryCursorMark = null; + do { + Element element = pidsCursorQuery(client, iterationUrl, masterQuery, cursorMark, 100, + filterQuery, "select", "compositeId+pid", "compositeId asc", "", ""); + cursorMark = findCursorMark(element); + queryCursorMark = findQueryCursorMark(element); + + try { + StringWriter wrt = new StringWriter(); + XMLUtils.print(element, wrt); + System.out.println(wrt); + } catch (TransformerException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + + List docs = XMLUtils.getElementsRecursive(element, new XMLUtils.ElementsFilter() { + + @Override + public boolean acceptElement(Element element) { + return element.getNodeName().equals("doc"); + } + }); + + List> pairs = docs.stream().map(doc-> { + + Element pidElm = XMLUtils.findElement(doc, new XMLUtils.ElementsFilter() { + + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + return (name != null && name.equals("pid")); + } + }); + + Element compositeIdElm = XMLUtils.findElement(doc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + return (name != null && name.equals("compositeId")); + } + }); + + + if (pidElm != null && compositeIdElm != null ) { + return Pair.of(pidElm.getTextContent(), compositeIdElm.getTextContent()); + } else return null; + }).filter(x -> x != null).collect(Collectors.toList()); + + allPids.addAll(pairs); + } while ((cursorMark != null && queryCursorMark != null) + && !cursorMark.equals(queryCursorMark)); + } catch (ParserConfigurationException | SAXException | IOException e) { + throw new RuntimeException(e); + } + } + return new ArrayList>(allPids); + } + + /** Render template */ + public static String renderTemplate(String api, Map iteration, Map destination) throws IOException { + String templatePath = String.format("reharvest_%s.xml", api); + + InputStream stream = KubernetesReharvestProcess.class.getResourceAsStream(templatePath); + + StringTemplate template = new StringTemplate( + org.apache.commons.io.IOUtils.toString(stream, "UTF-8"), DefaultTemplateLexer.class); + + template.setAttribute("iteration", iteration); + template.setAttribute("check", new HashMap<>()); + template.setAttribute("destination", destination); + template.setAttribute("timestamp", new HashMap<>()); + + String configuration = template.toString(); + return configuration; + } + + public static String fq(String api, String pid) { + switch(api) { + case "v7": return "root.pid:\""+pid+"\""; + case "v5": return "root_pid:\""+pid+"\""; + } + return "root_pid:\""+pid+"\""; + } + + public static void reharvestPIDFromGivenCollections(String pid, Map collectionConfigurations, String onlyShowConfiguration, Map destinationMap) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchMethodException, MigrateSolrIndexException, IOException, ParserConfigurationException, SAXException { + List harvestFiles = new ArrayList<>(); + for (String ac : collectionConfigurations.keySet()) { + try { + JSONObject colObject = collectionConfigurations.get(ac); + String apiVersion = colObject.optString("api","v5"); + if (!colObject.has("forwardurl")) { + LOGGER.severe(String.format("Skipping %s", ac)); + continue; + } + String channel = colObject.optString("forwardurl"); + + Map iteration = new HashMap<>(); + iteration.put("url", channel); + iteration.put("dl", ac); + iteration.put("fquery", fq(apiVersion, pid)); + + String configuration = renderTemplate( apiVersion, iteration, destinationMap); + File tmpFile = File.createTempFile(String.format("%s", ac), "reharvest"); + + Files.write(configuration.getBytes("UTF-8"), tmpFile); + harvestFiles.add(tmpFile); + + } catch (IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } + } + + for (File harvestFile : harvestFiles) { + if (ReharvestUtils.isOnlyShowConfiguration(onlyShowConfiguration)) { + String config = org.apache.commons.io.IOUtils.toString(new FileInputStream(harvestFile), "UTF-8"); + LOGGER.info(String.format("Configuration %s" ,config)); + } else { + // safra ?? + ParallelProcessImpl reharvest = new ParallelProcessImpl(); + String config = org.apache.commons.io.IOUtils.toString(new FileInputStream(harvestFile), "UTF-8"); + LOGGER.info(String.format("Configuration %s" ,config)); + reharvest.migrate(harvestFile); + } + + } + } + + public static boolean isOnlyShowConfiguration(String onlyShowConfiguration) { + if (onlyShowConfiguration != null && ("onlyshowconfiguration".equals(onlyShowConfiguration.toLowerCase()) || "true".equals(onlyShowConfiguration))) { + return true; + } + return false; + } + + +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java index 828369ae08..85a15a5db2 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java @@ -8,7 +8,7 @@ public interface ReharvestManager { public List getItems(); - public ReharvestItem getTopItem(); + public ReharvestItem getTopItem(String status); public ReharvestItem getItemById(String id); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index 5674253f25..f8f64b1ab8 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -13,6 +13,7 @@ import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; @@ -93,8 +94,8 @@ public Response register(String json) { @GET @Path("top") @Produces(MediaType.APPLICATION_JSON) - public Response getTopHarvest() { - ReharvestItem topItem = this.reharvestManager.getTopItem(); + public Response getTopHarvest(@QueryParam("state")String state) { + ReharvestItem topItem = this.reharvestManager.getTopItem(state); if (topItem != null) { return Response.ok(topItem.toJSON().toString()).build(); } else { @@ -102,19 +103,19 @@ public Response getTopHarvest() { } } - // -// -// @PUT -// @Path("{id}/state") -// @Produces(MediaType.APPLICATION_JSON) -// public Response getTopHarvest(@PathParam("id") String id, @QueryParam("status") String status) { -// ReharvestItem itemById = reharvestManager.getItemById(id); -// if (itemById != null) { -// itemById.setState(status); -// return Response.ok(itemById.toJSON().toString()).build(); -// } else { -// return Response.status(Response.Status.NOT_FOUND).build(); -// } -// } + + + @PUT + @Path("{id}/state") + @Produces(MediaType.APPLICATION_JSON) + public Response getTopHarvest(@PathParam("id") String id, @QueryParam("state") String state) { + ReharvestItem itemById = reharvestManager.getItemById(id); + if (itemById != null) { + itemById.setState(state); + return Response.ok(itemById.toJSON().toString()).build(); + } else { + return Response.status(Response.Status.NOT_FOUND).build(); + } + } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java index b2e8a19648..bfe7cd7d8c 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java @@ -4,6 +4,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.stream.Collectors; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; @@ -35,9 +36,10 @@ private void sortItems() { }); } - public ReharvestItem getTopItem() { + public ReharvestItem getTopItem(String status) { sortItems(); - return this.items.size() > 0 ? this.items.get(0) : null; + List listWithStatus = this.items.stream().filter(x -> x.getState() != null && x.getState().equals(status)).collect(Collectors.toList()); + return listWithStatus.size() > 0 ? listWithStatus.get(0) : null; } public ReharvestItem getItemById(String id) { From 3e46b3b41529b1e932503f55e3a38d333eb91388 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 29 Apr 2024 20:27:23 +0200 Subject: [PATCH 118/235] cdk reharvest process changes --- processes/cdkreharvest/build.gradle | 17 +++++++++++++++++ .../kramerius/KubernetesReharvestProcess.java | 2 +- .../incad/kramerius/utils/ReharvestUtils.java | 16 ++++++++-------- 3 files changed, 26 insertions(+), 9 deletions(-) diff --git a/processes/cdkreharvest/build.gradle b/processes/cdkreharvest/build.gradle index 0c319b3ed2..b64c93346d 100644 --- a/processes/cdkreharvest/build.gradle +++ b/processes/cdkreharvest/build.gradle @@ -1,3 +1,4 @@ +apply plugin: 'application' apply plugin: 'java' @@ -14,3 +15,19 @@ dependencies { testImplementation group: 'org.xmlunit', name: 'xmlunit-assertj', version: '2.8.4' } + + + +configurations.api { + exclude group: 'org.bouncycastle' + exclude group: 'com.lowagie' + exclude group: 'org.apache.pdfbox' + exclude group: 'com.levigo.jbig2' + exclude group: 'org.apache.solr:solr-solrj' + //exclude group: 'com.sun.jersey:jersey-json' +} + + +application { + mainClass = 'cz.incad.kramerius.KubernetesReharvestProcess' +} diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java index d571089348..f216ce04bd 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -62,7 +62,7 @@ public static void main(String[] args) throws ParserConfigurationException, Ille } // Top item - WebResource topWebResource = client.resource(wurl + "top"); + WebResource topWebResource = client.resource(wurl + "top?state=open"); ClientResponse topItemFrom = topWebResource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); if (topItemFrom.getStatus() == ClientResponse.Status.OK.getStatusCode()) { diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index c931224a68..aad3a2bdef 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -101,14 +101,14 @@ public static List> findAllPidsByGivenRootPid(Map docs = XMLUtils.getElementsRecursive(element, new XMLUtils.ElementsFilter() { From 784c061810964c260eeac0db583f69b2583bd2c8 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 1 May 2024 12:50:39 +0200 Subject: [PATCH 119/235] Reharvest - podname --- .../workers/replicate/ReplicateFinisher.java | 8 +- .../kramerius/KubernetesReharvestProcess.java | 47 +++++++++-- .../incad/kramerius/utils/ReharvestUtils.java | 50 +++++++----- .../admin/v10/reharvest/ReharvestItem.java | 79 +++++++++++++++---- .../v10/reharvest/ReharvestResource.java | 15 +++- .../incad/kramerius/timestamps/Timestamp.java | 37 ++++----- 6 files changed, 174 insertions(+), 62 deletions(-) diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateFinisher.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateFinisher.java index b41e263151..ceebc3060c 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateFinisher.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/workers/replicate/ReplicateFinisher.java @@ -63,11 +63,17 @@ public ReplicateFinisher(String timestampUrl, Element workerElm, Client client) } private JSONObject storeTimestamp() { - JSONObject jsonObject = new JSONObject(); + + String hostname = System.getenv("HOSTNAME"); + + + JSONObject jsonObject = new JSONObject(); jsonObject.put("workers", WORKERS.get()); jsonObject.put("batches", BATCHES.get()); jsonObject.put("indexed", NEWINDEXED); jsonObject.put("updated", UPDATED); + jsonObject.put("hostname", hostname); + if (typeOfCrawl != null) { jsonObject.put("type", typeOfCrawl); diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java index f216ce04bd..81ae2ac0c7 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -8,8 +8,13 @@ import java.util.TimeZone; import java.util.logging.Logger; +import javax.ws.rs.PUT; import javax.ws.rs.Path; +import javax.ws.rs.PathParam; +import javax.ws.rs.Produces; +import javax.ws.rs.QueryParam; import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; import javax.xml.parsers.ParserConfigurationException; import org.apache.commons.lang3.tuple.Pair; @@ -24,6 +29,7 @@ import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.json.JSONConfiguration; +import antlr.StringUtils; import cz.incad.kramerius.service.MigrateSolrIndexException; import cz.incad.kramerius.services.utils.kubernetes.KubernetesEnvSupport; import cz.incad.kramerius.utils.ReharvestUtils; @@ -71,6 +77,15 @@ public static void main(String[] args) throws ParserConfigurationException, Ille JSONObject itemObject = new JSONObject(t); String id = itemObject.getString("id"); JSONArray pids = itemObject.getJSONArray("pids"); + + if (!onlyShowConfiguration) { + changeState(client, wurl, id,"running"); + String podname = System.getProperty("HOSTNAME"); + if (cz.incad.kramerius.utils.StringUtils.isAnyString(podname)) { + changePodname(client, wurl, id, podname); + } + } + // find all pids List> allPidsList = ReharvestUtils.findAllPidsByGivenRootPid(iterationMap, client, pids); // delete all pids @@ -90,12 +105,7 @@ public static void main(String[] args) throws ParserConfigurationException, Ille } if (!onlyShowConfiguration) { - // @Path("{id}/state") - WebResource deleteWebResource = client.resource(wurl + id+"/state?state=closed"); - ClientResponse deleteResponse = deleteWebResource.accept(MediaType.APPLICATION_JSON).put(ClientResponse.class); - if (deleteResponse.getStatus() == ClientResponse.Status.OK.getStatusCode()) { - LOGGER.info("Reharvest item finished"); - } + changeState(client, wurl, id,"closed"); } } else { LOGGER.severe("No proxy configuration"); @@ -108,6 +118,29 @@ public static void main(String[] args) throws ParserConfigurationException, Ille } } + private static void changeState(Client client, String wurl, String id, String changeState) { + //String changeState = "closed"; + WebResource deleteWebResource = client.resource(wurl + id+"/state?state="+changeState); + ClientResponse deleteResponse = deleteWebResource.accept(MediaType.APPLICATION_JSON).put(ClientResponse.class); + if (deleteResponse.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + LOGGER.info("Reharvest item finished"); + } + } + +// @PUT +// @Path("{id}/pod") +// @Produces(MediaType.APPLICATION_JSON) +// public Response changePod(@PathParam("id") String id, @QueryParam("pod") String pod) { + + private static void changePodname(Client client, String wurl, String id, String podname) { + //String changeState = "closed"; + WebResource deleteWebResource = client.resource(wurl + id+"/pod?pod="+podname); + ClientResponse deleteResponse = deleteWebResource.accept(MediaType.APPLICATION_JSON).put(ClientResponse.class); + if (deleteResponse.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + LOGGER.info("Reharvest item finished"); + } + } + public static Map libraryConfigurations(Client client, String proxyURl) { Map configurations = new HashMap<>(); WebResource proxyWebResource = client.resource(proxyURl); @@ -118,7 +151,7 @@ public static Map libraryConfigurations(Client client, Strin JSONObject responseAllConnectedObject = new JSONObject(responseAllConnected); for (Object key : responseAllConnectedObject.keySet()) { JSONObject lib = responseAllConnectedObject.getJSONObject(key.toString()); - if (lib.has("status")) { + if (lib.has("status") && lib.getBoolean("status")) { String configURl = proxyURl; if (!configURl.endsWith("/")) { configURl += "/"; diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index aad3a2bdef..85de687754 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -49,7 +49,7 @@ private ReharvestUtils() {} /** Delete all given pids */ public static void deleteAllGivenPids(Client client,Map destinationMap, List> allPidsList, boolean onlyShowConfig) throws ParserConfigurationException { - int batchSize = 42; + int batchSize = 1000; int batches = allPidsList.size() / batchSize; if (allPidsList.size() % batchSize > 0) { batches = batches + 1; @@ -65,10 +65,9 @@ public static void deleteAllGivenPids(Client client,Map destinat idElm.setTextContent(pid.getRight().trim()); deleteBatch.getDocumentElement().appendChild(idElm); }); - try { - if (!onlyShowConfig) { + LOGGER.info("Deleting identifiers:"+batchPids); String destinationUrl = destinationMap.get("url")+"/update?commit=true"; String s = SolrUtils.sendToDest(destinationUrl, client, deleteBatch); } else { @@ -90,26 +89,19 @@ public static List> findAllPidsByGivenRootPid(Map docs = XMLUtils.getElementsRecursive(element, new XMLUtils.ElementsFilter() { @Override @@ -144,6 +136,8 @@ public boolean acceptElement(Element element) { }).filter(x -> x != null).collect(Collectors.toList()); allPids.addAll(pairs); + LOGGER.info(String.format( "Collected pids to delete %d", allPids.size())); + } while ((cursorMark != null && queryCursorMark != null) && !cursorMark.equals(queryCursorMark)); } catch (ParserConfigurationException | SAXException | IOException e) { @@ -190,9 +184,22 @@ public static void reharvestPIDFromGivenCollections(String pid, Map iteration = new HashMap<>(); - iteration.put("url", channel); + //http://mzk-tunnel.cdk-proxy.svc.cluster.local/search" + //http://knav-tunnel.cdk-proxy.svc.cluster.local/search/api/v5.0/cdk/forward/sync/solr + //v7.0 + //search/api/cdk/v7.0/forward/sync/solr/ + if (apiVersion.toLowerCase().equals("v5")) { + //channel = + iteration.put("url", channel+(channel.endsWith("/") ? "" : "/")+"api/v5.0/cdk/forward/sync/solr"); + } else { + iteration.put("url", channel+(channel.endsWith("/") ? "" : "/")+"api/cdk/v7.0/forward/sync/solr"); + } + + + iteration.put("dl", ac); iteration.put("fquery", fq(apiVersion, pid)); @@ -213,12 +220,17 @@ public static void reharvestPIDFromGivenCollections(String pid, Map pids= new ArrayList<>(); private Instant timestamp = Instant.now(); - + private String podname; + + private List libraries = new ArrayList<>(); + public ReharvestItem(String id, String name, String state, List pids) { super(); @@ -71,7 +87,15 @@ public List getPids() { public void setPids(List pids) { this.pids = pids; } - + + public List getLibraries() { + return libraries; + } + + public void setLibraries(List libraries) { + this.libraries = libraries; + } + public Instant getTimestamp() { return timestamp; } @@ -88,37 +112,56 @@ public void setType(String type) { this.type = type; } + + public void setPodname(String podname) { + this.podname = podname; + } + + public String getPodname() { + return podname; + } + + public JSONObject toJSON() { JSONObject obj = new JSONObject(); - obj.put("id", this.id); - obj.put("name", this.name); + obj.put(ID_KEYWORD, this.id); + obj.put(NAME_KEYWORD, this.name); if (this.state != null) { - obj.put("state", this.state); + obj.put(STATE_KEYWORD, this.state); } if (this.type != null) { - obj.put("type", this.type); + obj.put(TYPE_KEYWORD, this.type); } JSONArray jsonArray = new JSONArray(); this.pids.forEach(jsonArray::put); - obj.put("pids", jsonArray); + obj.put(PIDS_KEYWORD, jsonArray); - obj.put("timestamp",DateTimeFormatter.ISO_INSTANT.format(this.timestamp)); + if (!this.libraries.isEmpty()) { + JSONArray libsArray = new JSONArray(); + this.libraries.forEach(libsArray::put); + } + + if (this.podname != null) { + obj.put(POD_NAME_KEYWORD, this.podname); + } + + obj.put(TIMESTAMP_KEYWORD,DateTimeFormatter.ISO_INSTANT.format(this.timestamp)); return obj; } public static ReharvestItem fromJSON(JSONObject json) throws ParseException { - String id = json.getString("id"); - String name= json.getString("name"); - JSONArray array= json.getJSONArray("pids"); + String id = json.getString(ID_KEYWORD); + String name= json.getString(NAME_KEYWORD); + JSONArray array= json.getJSONArray(PIDS_KEYWORD); List pids = new ArrayList<>(); for (int i = 0; i < array.length(); i++) { pids.add(array.getString(i)); } - String state = json.optString("state"); - String type = json.optString("type"); + String state = json.optString(STATE_KEYWORD); + String type = json.optString(TYPE_KEYWORD); ReharvestItem item = new ReharvestItem(id); item.setName(name); item.setPids(pids); @@ -126,11 +169,15 @@ public static ReharvestItem fromJSON(JSONObject json) throws ParseException { item.setState(StringUtils.isAnyString(state) ? state : "open"); item.setType(StringUtils.isAnyString(type) ? type : "root.pid"); - if (json.has("timestamp")) { - String timestamp = json.getString("timestamp"); + if (json.has(TIMESTAMP_KEYWORD)) { + String timestamp = json.getString(TIMESTAMP_KEYWORD); item.setTimestamp( Instant.from(DateTimeFormatter.ISO_INSTANT.parse(timestamp))); } + if (json.has(POD_NAME_KEYWORD)) { + item.setPodname(json.getString(POD_NAME_KEYWORD)); + } + return item; } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index f8f64b1ab8..8201a4484a 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -108,7 +108,7 @@ public Response getTopHarvest(@QueryParam("state")String state) { @PUT @Path("{id}/state") @Produces(MediaType.APPLICATION_JSON) - public Response getTopHarvest(@PathParam("id") String id, @QueryParam("state") String state) { + public Response changeState(@PathParam("id") String id, @QueryParam("state") String state) { ReharvestItem itemById = reharvestManager.getItemById(id); if (itemById != null) { itemById.setState(state); @@ -118,4 +118,17 @@ public Response getTopHarvest(@PathParam("id") String id, @QueryParam("state") S } } + @PUT + @Path("{id}/pod") + @Produces(MediaType.APPLICATION_JSON) + public Response changePod(@PathParam("id") String id, @QueryParam("pod") String pod) { + ReharvestItem itemById = reharvestManager.getItemById(id); + if (itemById != null) { + itemById.setPodname(pod); + return Response.ok(itemById.toJSON().toString()).build(); + } else { + return Response.status(Response.Status.NOT_FOUND).build(); + } + } + } diff --git a/shared/common/src/main/java/cz/incad/kramerius/timestamps/Timestamp.java b/shared/common/src/main/java/cz/incad/kramerius/timestamps/Timestamp.java index a7823eca4c..f094b84f75 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/timestamps/Timestamp.java +++ b/shared/common/src/main/java/cz/incad/kramerius/timestamps/Timestamp.java @@ -5,28 +5,29 @@ import org.apache.solr.common.SolrInputDocument; import org.json.JSONObject; +//TODO: Pod name ?? public interface Timestamp { - - public String getType(); - - public String getName(); - - public Date getDate(); - - public long getIndexed(); - - public long getUpdated(); - - public long getBatches(); - - public long getWorkers(); - - public String getId(); + + public String getType(); + + public String getName(); + + public Date getDate(); + + public long getIndexed(); + + public long getUpdated(); + + public long getBatches(); + + public long getWorkers(); + + public String getId(); public SolrInputDocument toSolrDoc(); - + public JSONObject toJSONObject(); - + public void updateDate(Date date); public void updateName(String name); From cc1a288c504a80e994774da2166ea3815edf0cc5 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 1 May 2024 12:52:05 +0200 Subject: [PATCH 120/235] Version changed --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 813b689860..298e304ed2 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ #org.gradle.java.home=c:\\Users\\happy\\Programs\\jdk18 group=cz.incad.kramerius -version=7.0.33_dev17_cdk +version=7.0.33_dev18_cdk From 1b6146b3819fabd530cc32ee23b39db15011ff98 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 1 May 2024 15:11:20 +0200 Subject: [PATCH 121/235] Reharvesting; - 404 status --- .../kramerius/KubernetesReharvestProcess.java | 4 +- .../DefaultOnePropertiesInstance.java | 19 ++++---- .../DefaultPropertiesInstances.java | 11 +++-- .../v60/redirection/ProxyHandlerSupport.java | 45 +++++++++++++++++-- .../redirection/item/ProxyItemHandler.java | 5 ++- .../redirection/item/V5ForwardHandler.java | 19 ++++---- .../redirection/item/V5RedirectHandler.java | 5 ++- .../redirection/item/V7ForwardHandler.java | 21 +++++---- .../redirection/item/V7RedirectHandler.java | 6 ++- .../redirection/user/ProxyUserHandler.java | 5 ++- .../user/V5ForwardUserHandler.java | 5 ++- .../user/V7ForwardUserHandler.java | 5 ++- .../client/v60/filter/DefaultFilterTest.java | 14 ++++-- 13 files changed, 111 insertions(+), 53 deletions(-) diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java index 81ae2ac0c7..a672bebf41 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -123,7 +123,7 @@ private static void changeState(Client client, String wurl, String id, String ch WebResource deleteWebResource = client.resource(wurl + id+"/state?state="+changeState); ClientResponse deleteResponse = deleteWebResource.accept(MediaType.APPLICATION_JSON).put(ClientResponse.class); if (deleteResponse.getStatus() == ClientResponse.Status.OK.getStatusCode()) { - LOGGER.info("Reharvest item finished"); + LOGGER.info(String.format("Change state for %s -> %s ", id, changeState)); } } @@ -137,7 +137,7 @@ private static void changePodname(Client client, String wurl, String id, String WebResource deleteWebResource = client.resource(wurl + id+"/pod?pod="+podname); ClientResponse deleteResponse = deleteWebResource.accept(MediaType.APPLICATION_JSON).put(ClientResponse.class); if (deleteResponse.getStatus() == ClientResponse.Status.OK.getStatusCode()) { - LOGGER.info("Reharvest item finished"); + LOGGER.info(String.format("Change podname for %s -> %s ", id, podname)); } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultOnePropertiesInstance.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultOnePropertiesInstance.java index 9a870427df..4d4150d5a7 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultOnePropertiesInstance.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultOnePropertiesInstance.java @@ -6,6 +6,7 @@ import com.sun.jersey.api.client.Client; import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance.TypeOfChangedStatus; @@ -23,14 +24,16 @@ public class DefaultOnePropertiesInstance implements OneInstance { private Instances instances; + private ReharvestManager reharvestManager; private String instanceAcronym; private boolean connected = true; private TypeOfChangedStatus typeOfChangedStatus = TypeOfChangedStatus.automat; private Map info = new HashMap<>(); - public DefaultOnePropertiesInstance(Instances instances, String instanceAcronym) { + public DefaultOnePropertiesInstance(ReharvestManager reharvestManager, Instances instances, String instanceAcronym) { super(); + this.reharvestManager = reharvestManager; this.instanceAcronym = instanceAcronym; this.instances = instances; } @@ -61,19 +64,19 @@ public ProxyItemHandler createProxyItemHandler(User user, Client client, SolrAcc InstanceType instanceType = getInstanceType(); switch (instanceType) { case V5: - return new V5ForwardHandler(this.instances, user, client, solrAccess, source, pid, remoteAddr); + return new V5ForwardHandler(this.reharvestManager, this.instances, user, client, solrAccess, source, pid, remoteAddr); default: - return new V7ForwardHandler(this.instances, user, client, solrAccess, source, pid, remoteAddr); + return new V7ForwardHandler(this.reharvestManager,this.instances, user, client, solrAccess, source, pid, remoteAddr); } } else { InstanceType instanceType = getInstanceType(); switch (instanceType) { case V5: - return new V5RedirectHandler(this.instances, user, client, solrAccess, source, pid, remoteAddr); + return new V5RedirectHandler(this.reharvestManager,this.instances, user, client, solrAccess, source, pid, remoteAddr); case V7: - return new V7RedirectHandler(this.instances, user, client, solrAccess, source, pid, remoteAddr); + return new V7RedirectHandler(this.reharvestManager,this.instances, user, client, solrAccess, source, pid, remoteAddr); default: - return new V5RedirectHandler(this.instances, user, client, solrAccess, source, pid, remoteAddr); + return new V5RedirectHandler(this.reharvestManager,this.instances, user, client, solrAccess, source, pid, remoteAddr); } } @@ -86,9 +89,9 @@ public ProxyUserHandler createProxyUserHandler(User user, Client client, SolrAcc InstanceType instanceType = getInstanceType(); switch (instanceType) { case V5: - return new V5ForwardUserHandler(this.instances, user, client, solrAccess, source, remoteAddr); + return new V5ForwardUserHandler(this.reharvestManager,this.instances, user, client, solrAccess, source, remoteAddr); default: - return new V7ForwardUserHandler(this.instances, user, client, solrAccess, source, remoteAddr); + return new V7ForwardUserHandler(this.reharvestManager,this.instances, user, client, solrAccess, source, remoteAddr); } } else { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java index db49471c13..5876a50078 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java @@ -14,6 +14,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; +import javax.inject.Inject; import javax.ws.rs.core.MediaType; import org.apache.commons.configuration.Configuration; @@ -26,6 +27,7 @@ import com.sun.jersey.api.client.UniformInterfaceException; import com.sun.jersey.api.client.WebResource; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance.TypeOfChangedStatus; @@ -41,10 +43,12 @@ public class DefaultPropertiesInstances implements Instances { private List instances = new ArrayList<>(); private Set names = new HashSet<>(); + private ReharvestManager reharvestManager; - public DefaultPropertiesInstances() { + @Inject + public DefaultPropertiesInstances(ReharvestManager reharvestManager) { super(); - + this.reharvestManager = reharvestManager; LOGGER.info("Refreshing configuration "); refreshingConfiguration(); } @@ -69,10 +73,9 @@ protected void refreshingConfiguration() { private void addOneInstance(String acronym) { LOGGER.info(String.format("Adding library %s", acronym)); names.add(acronym); - DefaultOnePropertiesInstance di = new DefaultOnePropertiesInstance(this, acronym); + DefaultOnePropertiesInstance di = new DefaultOnePropertiesInstance(this.reharvestManager, this, acronym); instances.add(di); } - @Override public List allInstances() { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 00e1122f09..7d3816f451 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -6,8 +6,11 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; +import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; @@ -22,15 +25,20 @@ import javax.ws.rs.core.StreamingOutput; import org.apache.commons.io.IOUtils; +import org.w3c.dom.Document; +import org.w3c.dom.Element; import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.StringUtils; +import cz.incad.kramerius.utils.XMLUtils; import cz.incad.kramerius.utils.conf.KConfiguration; public abstract class ProxyHandlerSupport { @@ -45,9 +53,11 @@ public abstract class ProxyHandlerSupport { protected User user; protected String remoteAddr; protected Instances instances; + protected ReharvestManager reharvestManager; - public ProxyHandlerSupport(Instances instances, User user, Client client, SolrAccess solrAccess, String source, + public ProxyHandlerSupport(ReharvestManager reharvestManager, Instances instances, User user, Client client, SolrAccess solrAccess, String source, String remoteAddr) { + this.reharvestManager = reharvestManager; this.source = source; this.client = client; this.solrAccess = solrAccess; @@ -92,10 +102,15 @@ public ClientResponse forwardedResponse(String url) throws ProxyHandlerException } public Response buildForwardResponseGET(String url) throws ProxyHandlerException { - return buildForwardResponseGET(url, null); + return buildForwardResponseGET(url, null, null); } + + public Response buildForwardResponseGET(String url, String pid) throws ProxyHandlerException { + return buildForwardResponseGET(url, null, pid); + } + - public Response buildForwardResponseGET(String url, String mimetype) throws ProxyHandlerException { + public Response buildForwardResponseGET(String url, String mimetype, String pid) throws ProxyHandlerException { WebResource.Builder b = buidFowrardResponse(url); ClientResponse response = b.get(ClientResponse.class); if (response.getStatus() == 200) { @@ -132,6 +147,30 @@ public void write(OutputStream output) throws IOException, WebApplicationExcepti return respEntity.build(); } else { + // event for reharvest + if (response.getStatus() == 404) { + if (reharvestManager != null) { + try { + Document solrDataByPid = this.solrAccess.getSolrDataByPid(pid); + Element rootPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("src")) { + String fieldName = element.getAttribute("name"); + return fieldName.equals("root.pid"); + } + return false; + } + }); + if (rootPid != null) { + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Reharvest from CDK Core","open", new ArrayList<>(Arrays.asList(rootPid.getTextContent().trim()))); + this.reharvestManager.register(reharvestItem); + } + } catch (IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage()); + } + } + } return Response.status(response.getStatus()).build(); } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java index f4e2d9daa0..01d1d84e03 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java @@ -14,6 +14,7 @@ import cz.incad.kramerius.SolrAccess; import cz.incad.kramerius.fedora.om.RepositoryException; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerSupport; @@ -38,8 +39,8 @@ public static enum RequestMethodName { protected String pid; - public ProxyItemHandler(Instances instances, User user, Client client, SolrAccess solrAccess, String source, String pid, String remoteAddr) { - super(instances,user,client,solrAccess,source, remoteAddr); + public ProxyItemHandler(ReharvestManager reharvestManager, Instances instances, User user, Client client, SolrAccess solrAccess, String source, String pid, String remoteAddr) { + super(reharvestManager, instances,user,client,solrAccess,source, remoteAddr); this.source = source; this.pid = pid; } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java index ae6e4f7da3..74674b8c8a 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java @@ -23,6 +23,7 @@ import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.filter.ProxyFilter; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; @@ -43,9 +44,9 @@ protected String forwardUrl() { } - public V5ForwardHandler(Instances instances, User user, Client client, SolrAccess solrAccess, String source, + public V5ForwardHandler(ReharvestManager reharvestManager, Instances instances, User user, Client client, SolrAccess solrAccess, String source, String pid, String remoteAddr) { - super(instances, user, client, solrAccess, source, pid, remoteAddr); + super(reharvestManager, instances, user, client, solrAccess, source, pid, remoteAddr); } @Override @@ -56,7 +57,7 @@ public Response image(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url); + return buildForwardResponseGET(url, this.pid); } } @@ -69,7 +70,7 @@ public Response imagePreview(RequestMethodName method) throws ProxyHandlerExcept if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url); + return buildForwardResponseGET(url, this.pid); } } @@ -86,7 +87,7 @@ public Response mods(RequestMethodName method) throws ProxyHandlerException { + "/streams/BIBLIO_MODS"; // String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/item/" // + this.pid + "/streams/BIBLIO_MODS"; - return buildForwardResponseGET(url); + return buildForwardResponseGET(url, this.pid); } } @@ -99,7 +100,7 @@ public Response zoomifyImageProperties(RequestMethodName method) throws ProxyHan if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url); + return buildForwardResponseGET(url, this.pid); } } @@ -108,7 +109,7 @@ public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHan String baseurl = this.forwardUrl(); String formatted = String.format("api/v5.0/cdk/forward/zoomify/%s/%s/%s", this.pid, tileGroupStr, tileStr); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + formatted; - return buildForwardResponseGET(url); + return buildForwardResponseGET(url, this.pid); } @Override @@ -120,7 +121,7 @@ public Response textOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url); + return buildForwardResponseGET(url, this.pid); } } @@ -133,7 +134,7 @@ public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url); + return buildForwardResponseGET(url, this.pid); } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java index ca3b50f7fe..30df4c0539 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java @@ -33,6 +33,7 @@ import cz.incad.kramerius.repository.KrameriusRepositoryApi.FosterRelationsMapping; import cz.incad.kramerius.repository.KrameriusRepositoryApi.KnownRelations; import cz.incad.kramerius.repository.KrameriusRepositoryApi.OwnRelationsMapping; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.ProxyItemHandler.RequestMethodName; @@ -44,9 +45,9 @@ public class V5RedirectHandler extends ProxyItemHandler { public static final Logger LOGGER = Logger.getLogger(V5RedirectHandler.class.getName()); - public V5RedirectHandler(Instances instances, User user, Client client, SolrAccess solrAccess, String source, + public V5RedirectHandler(ReharvestManager reharvestManager, Instances instances, User user, Client client, SolrAccess solrAccess, String source, String pid, String remoteAddr) { - super(instances, user, client, solrAccess, source, pid, remoteAddr); + super(reharvestManager, instances, user, client, solrAccess, source, pid, remoteAddr); } @Override diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index f51cf2dd1e..f33b9c2c86 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -24,6 +24,7 @@ import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.ProxyItemHandler.RequestMethodName; @@ -34,8 +35,8 @@ public class V7ForwardHandler extends V7RedirectHandler { public static final Logger LOGGER = Logger.getLogger(V5ForwardHandler.class.getName()); - public V7ForwardHandler(Instances instances, User user, Client client, SolrAccess solrAccess, String source, String pid, String remoteAddr) { - super(instances, user, client, solrAccess, source, pid, remoteAddr); + public V7ForwardHandler(ReharvestManager reharvestManager, Instances instances, User user, Client client, SolrAccess solrAccess, String source, String pid, String remoteAddr) { + super(reharvestManager,instances, user, client, solrAccess, source, pid, remoteAddr); } protected String forwardUrl() { @@ -45,8 +46,6 @@ protected String forwardUrl() { } - - @Override public Response info() throws ProxyHandlerException { @@ -97,7 +96,7 @@ public Response image(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url); + return buildForwardResponseGET(url, this.pid); } } @@ -110,7 +109,7 @@ public Response imagePreview(RequestMethodName method) throws ProxyHandlerExcept if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url); + return buildForwardResponseGET(url, this.pid); } } @@ -122,7 +121,7 @@ public Response mods(RequestMethodName method) throws ProxyHandlerException { String baseurl = this.forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/item/" + this.pid + "/streams/BIBLIO_MODS"; - return buildForwardResponseGET(url); + return buildForwardResponseGET(url,this.pid); } } @@ -135,7 +134,7 @@ public Response zoomifyImageProperties(RequestMethodName method) throws ProxyHan String baseurl = forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/zoomify/" + this.pid + "/ImageProperties.xml"; - return buildForwardResponseGET(url); + return buildForwardResponseGET(url, this.pid); } } @@ -146,7 +145,7 @@ public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHan String baseurl = forwardUrl(); String formatted = String.format("api/cdk/v7.0/forward/zoomify/%s/%s/%s", this.pid, tileGroupStr, tileStr); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + formatted; - return buildForwardResponseGET(url); + return buildForwardResponseGET(url, this.pid); } @@ -159,7 +158,7 @@ public Response textOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url); + return buildForwardResponseGET(url, this.pid); } } @@ -171,7 +170,7 @@ public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url); + return buildForwardResponseGET(url, this.pid); } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java index 11994cd20f..5204b26c1d 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java @@ -13,6 +13,8 @@ import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.ProxyItemHandler.RequestMethodName; @@ -23,8 +25,8 @@ public class V7RedirectHandler extends ProxyItemHandler{ public static final Logger LOGGER = Logger.getLogger(V7RedirectHandler.class.getName()); - public V7RedirectHandler(Instances instances, User user, Client client, SolrAccess solrAccess, String source, String pid,String remoteAddr) { - super(instances, user,client, solrAccess, source, pid, remoteAddr); + public V7RedirectHandler(ReharvestManager reharvestManager, Instances instances, User user, Client client, SolrAccess solrAccess, String source, String pid,String remoteAddr) { + super(reharvestManager, instances, user,client, solrAccess, source, pid, remoteAddr); } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/ProxyUserHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/ProxyUserHandler.java index d82e71e0f4..8d54f27e65 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/ProxyUserHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/ProxyUserHandler.java @@ -7,6 +7,7 @@ import com.sun.jersey.api.client.Client; import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerSupport; @@ -14,9 +15,9 @@ public abstract class ProxyUserHandler extends ProxyHandlerSupport { - public ProxyUserHandler(Instances instances, User user, Client client, SolrAccess solrAccess, String source, + public ProxyUserHandler(ReharvestManager reharvestManager, Instances instances, User user, Client client, SolrAccess solrAccess, String source, String remoteAddr) { - super(instances, user, client, solrAccess, source, remoteAddr); + super(reharvestManager, instances, user, client, solrAccess, source, remoteAddr); } public abstract Pair> user() throws ProxyHandlerException; diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java index 5194ed3e6d..53d07b793d 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java @@ -16,6 +16,7 @@ import com.sun.jersey.api.client.ClientResponse; import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; import cz.incad.kramerius.security.Role; @@ -26,9 +27,9 @@ public class V5ForwardUserHandler extends ProxyUserHandler { - public V5ForwardUserHandler(Instances instances, User user, Client client, SolrAccess solrAccess, String source, + public V5ForwardUserHandler(ReharvestManager reharvestManager, Instances instances, User user, Client client, SolrAccess solrAccess, String source, String remoteAddr) { - super(instances, user, client, solrAccess, source, remoteAddr); + super(reharvestManager,instances, user, client, solrAccess, source, remoteAddr); } protected String forwardUrl() { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java index 359c343b86..655adfe1d1 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java @@ -14,6 +14,7 @@ import com.sun.jersey.api.client.ClientResponse; import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; import cz.incad.kramerius.security.Role; @@ -31,9 +32,9 @@ public class V7ForwardUserHandler extends ProxyUserHandler { private static final String AUTHENTICATED_KEY = "authenticated"; private static final String UID_KEY = "uid"; - public V7ForwardUserHandler(Instances instances, User user, Client client, SolrAccess solrAccess, String source, + public V7ForwardUserHandler(ReharvestManager reharvestManager, Instances instances, User user, Client client, SolrAccess solrAccess, String source, String remoteAddr) { - super(instances, user, client, solrAccess, source, remoteAddr); + super(reharvestManager, instances, user, client, solrAccess, source, remoteAddr); } protected String forwardUrl() { diff --git a/rest/src/test/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilterTest.java b/rest/src/test/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilterTest.java index 7f6524ca72..3f87da3ad4 100644 --- a/rest/src/test/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilterTest.java +++ b/rest/src/test/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilterTest.java @@ -27,6 +27,8 @@ import org.w3c.dom.Element; import org.xml.sax.SAXException; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.impl.MemoryReharvestManagerImpl; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance.TypeOfChangedStatus; import cz.incad.kramerius.rest.apiNew.client.v60.libs.properties.DefaultOnePropertiesInstance; @@ -336,8 +338,12 @@ public boolean acceptElement(Element element) { @Test public void testValueDocWithPhysicalLocationJSON() throws IOException { + + + ReharvestManager reharvest = new MemoryReharvestManagerImpl(); + DefaultPropertiesInstances inst = createMockBuilder(DefaultPropertiesInstances.class) - .withConstructor() + .withConstructor(reharvest) .addMockedMethod("allInstances") .addMockedMethod("enabledInstances") .addMockedMethod("disabledInstances") @@ -346,9 +352,9 @@ public void testValueDocWithPhysicalLocationJSON() throws IOException { .createMock(); - DefaultOnePropertiesInstance mzk = new DefaultOnePropertiesInstance(inst,"mzk"); - DefaultOnePropertiesInstance nkp = new DefaultOnePropertiesInstance(inst,"nkp"); - DefaultOnePropertiesInstance knav = new DefaultOnePropertiesInstance(inst,"knav"); + DefaultOnePropertiesInstance mzk = new DefaultOnePropertiesInstance(null, inst,"mzk"); + DefaultOnePropertiesInstance nkp = new DefaultOnePropertiesInstance(null, inst,"nkp"); + DefaultOnePropertiesInstance knav = new DefaultOnePropertiesInstance(null,inst,"knav"); knav.setConnected(false, TypeOfChangedStatus.user); From 3c6fb9ecbb7352513f18c5c114536453edcd1f74 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 1 May 2024 15:33:06 +0200 Subject: [PATCH 122/235] Reharvest - 404; --- .../AlreadyRegistedPidsException.java | 17 +++++++++++++++++ .../admin/v10/reharvest/ReharvestManager.java | 4 +--- .../admin/v10/reharvest/ReharvestResource.java | 4 ++++ .../impl/MemoryReharvestManagerImpl.java | 18 ++++++++++++++---- .../v60/redirection/ProxyHandlerSupport.java | 12 ++++++++++-- 5 files changed, 46 insertions(+), 9 deletions(-) create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/AlreadyRegistedPidsException.java diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/AlreadyRegistedPidsException.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/AlreadyRegistedPidsException.java new file mode 100644 index 0000000000..e10827ce0d --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/AlreadyRegistedPidsException.java @@ -0,0 +1,17 @@ +package cz.incad.kramerius.rest.apiNew.admin.v10.reharvest; + +import java.util.List; + +public class AlreadyRegistedPidsException extends Exception { + + private List pids; + + public AlreadyRegistedPidsException(List pids) { + super(); + this.pids = pids; + } + + public List getPids() { + return pids; + } +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java index 85a15a5db2..50214d96f2 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java @@ -4,15 +4,13 @@ public interface ReharvestManager { - public void register(ReharvestItem item); + public void register(ReharvestItem item) throws AlreadyRegistedPidsException; public List getItems(); public ReharvestItem getTopItem(String status); - public ReharvestItem getItemById(String id); - public void deregister(String id); } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index 8201a4484a..5f5d43cb00 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -23,6 +23,8 @@ import com.google.inject.Inject; +import cz.incad.kramerius.rest.apiNew.exceptions.BadRequestException; + @Path("/admin/v7.0/reharvest") public class ReharvestResource { @@ -86,6 +88,8 @@ public Response register(String json) { } catch (JSONException | ParseException e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); + } catch(AlreadyRegistedPidsException e) { + throw new BadRequestException(" Request contains already registered pids "+e.getPids()); } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java index bfe7cd7d8c..81c2c49f9f 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java @@ -6,6 +6,7 @@ import java.util.Map; import java.util.stream.Collectors; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.AlreadyRegistedPidsException; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; @@ -19,10 +20,19 @@ public MemoryReharvestManagerImpl() { super(); } - public void register(ReharvestItem item) { - this.items.add(item); - sortItems(); - this.mapper.put(item.getId(), item); + public void register(ReharvestItem item) throws AlreadyRegistedPidsException { + List alreadyRegistredPids = this.items.stream().filter(x -> x.getState() != null && x.getState().equals("open")).map(ReharvestItem::getPids).flatMap(List::stream).collect(Collectors.toList()); + List intersection = new ArrayList<>(item.getPids()); + intersection.retainAll(alreadyRegistredPids); + + if (intersection.isEmpty()) { + this.items.add(item); + sortItems(); + this.mapper.put(item.getId(), item); + } else { + throw new AlreadyRegistedPidsException(intersection); + } + } public List getItems() { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 7d3816f451..42f81d8347 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -25,6 +25,7 @@ import javax.ws.rs.core.StreamingOutput; import org.apache.commons.io.IOUtils; +import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -33,6 +34,7 @@ import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.AlreadyRegistedPidsException; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; @@ -163,8 +165,14 @@ public boolean acceptElement(Element element) { } }); if (rootPid != null) { - ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Reharvest from CDK Core","open", new ArrayList<>(Arrays.asList(rootPid.getTextContent().trim()))); - this.reharvestManager.register(reharvestItem); + try { + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Reharvest from CDK Core","open", new ArrayList<>(Arrays.asList(rootPid.getTextContent().trim()))); + this.reharvestManager.register(reharvestItem); + } catch (DOMException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } catch (AlreadyRegistedPidsException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } } } catch (IOException e) { LOGGER.log(Level.SEVERE,e.getMessage()); From 75f63d0ca036f9974a98f928eb1a0caa253ef905 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 1 May 2024 15:49:19 +0200 Subject: [PATCH 123/235] Version --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 298e304ed2..feae72ee9e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ #org.gradle.java.home=c:\\Users\\happy\\Programs\\jdk18 group=cz.incad.kramerius -version=7.0.33_dev18_cdk +version=7.0.33_dev19_cdk From ea7413bfd1eec86a756ae362b72244e2c1e7577d Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 1 May 2024 16:21:01 +0200 Subject: [PATCH 124/235] Info messages --- .../v60/libs/properties/DefaultPropertiesInstances.java | 4 ++-- .../apiNew/client/v60/redirection/ProxyHandlerSupport.java | 6 +++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java index 5876a50078..6f73ea58f7 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/libs/properties/DefaultPropertiesInstances.java @@ -49,7 +49,7 @@ public class DefaultPropertiesInstances implements Instances { public DefaultPropertiesInstances(ReharvestManager reharvestManager) { super(); this.reharvestManager = reharvestManager; - LOGGER.info("Refreshing configuration "); + LOGGER.info("Refreshing configuration with reharvestManager "+this.reharvestManager); refreshingConfiguration(); } @@ -71,7 +71,7 @@ protected void refreshingConfiguration() { } private void addOneInstance(String acronym) { - LOGGER.info(String.format("Adding library %s", acronym)); + LOGGER.info(String.format("Adding library %s with reharvestManager %s", acronym, reharvestManager.toString())); names.add(acronym); DefaultOnePropertiesInstance di = new DefaultOnePropertiesInstance(this.reharvestManager, this, acronym); instances.add(di); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 42f81d8347..2c78e78954 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -157,7 +157,7 @@ public void write(OutputStream output) throws IOException, WebApplicationExcepti Element rootPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { @Override public boolean acceptElement(Element element) { - if (element.getNodeName().equals("src")) { + if (element.getNodeName().equals("str")) { String fieldName = element.getAttribute("name"); return fieldName.equals("root.pid"); } @@ -173,10 +173,14 @@ public boolean acceptElement(Element element) { } catch (AlreadyRegistedPidsException e) { LOGGER.log(Level.SEVERE,e.getMessage(),e); } + } else { + LOGGER.log(Level.SEVERE, "Cannot find root.pid element"); } } catch (IOException e) { LOGGER.log(Level.SEVERE,e.getMessage()); } + } else { + LOGGER.log(Level.SEVERE,"No reharvest manager"); } } return Response.status(response.getStatus()).build(); From 1a88c4282d01d4a628769775256eed2e6cd28d22 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 1 May 2024 16:21:51 +0200 Subject: [PATCH 125/235] Version --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index feae72ee9e..11d59301d0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,4 @@ #org.gradle.java.home=c:\\Users\\happy\\Programs\\jdk18 group=cz.incad.kramerius -version=7.0.33_dev19_cdk +version=7.0.33_dev20_cdk From 2d3e95f635dcdf2a8b008eee298b40b6ef0efbd2 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 1 May 2024 16:31:26 +0200 Subject: [PATCH 126/235] omitting streams ALTO and TEXT_OCR --- .../apiNew/client/v60/redirection/ProxyHandlerSupport.java | 4 ++-- .../client/v60/redirection/item/V5ForwardHandler.java | 6 +++--- .../client/v60/redirection/item/V7ForwardHandler.java | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 2c78e78954..3089f58405 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -151,7 +151,7 @@ public void write(OutputStream output) throws IOException, WebApplicationExcepti } else { // event for reharvest if (response.getStatus() == 404) { - if (reharvestManager != null) { + if (reharvestManager != null && pid != null) { try { Document solrDataByPid = this.solrAccess.getSolrDataByPid(pid); Element rootPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { @@ -180,7 +180,7 @@ public boolean acceptElement(Element element) { LOGGER.log(Level.SEVERE,e.getMessage()); } } else { - LOGGER.log(Level.SEVERE,"No reharvest manager"); + LOGGER.log(Level.SEVERE,"No reharvest manager or pid "); } } return Response.status(response.getStatus()).build(); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java index 74674b8c8a..72907cd56c 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java @@ -70,7 +70,7 @@ public Response imagePreview(RequestMethodName method) throws ProxyHandlerExcept if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, this.pid); + return buildForwardResponseGET(url, null); } } @@ -121,7 +121,7 @@ public Response textOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, this.pid); + return buildForwardResponseGET(url,null); } } @@ -134,7 +134,7 @@ public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, this.pid); + return buildForwardResponseGET(url, null); } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index f33b9c2c86..54d66f9d12 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -109,7 +109,7 @@ public Response imagePreview(RequestMethodName method) throws ProxyHandlerExcept if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, this.pid); + return buildForwardResponseGET(url, null); } } @@ -158,7 +158,7 @@ public Response textOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, this.pid); + return buildForwardResponseGET(url, null); } } @@ -170,7 +170,7 @@ public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, this.pid); + return buildForwardResponseGET(url, null); } } From ce6c6e3c67252e1b73477f46e50c80c5d379d2e7 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 1 May 2024 19:24:54 +0200 Subject: [PATCH 127/235] Message changed --- .../java/cz/incad/kramerius/KubernetesReharvestProcess.java | 6 +++--- .../apiNew/client/v60/redirection/ProxyHandlerSupport.java | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java index a672bebf41..fc07169e3d 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -80,7 +80,7 @@ public static void main(String[] args) throws ParserConfigurationException, Ille if (!onlyShowConfiguration) { changeState(client, wurl, id,"running"); - String podname = System.getProperty("HOSTNAME"); + String podname = env.get("HOSTNAME"); if (cz.incad.kramerius.utils.StringUtils.isAnyString(podname)) { changePodname(client, wurl, id, podname); } @@ -134,8 +134,8 @@ private static void changeState(Client client, String wurl, String id, String ch private static void changePodname(Client client, String wurl, String id, String podname) { //String changeState = "closed"; - WebResource deleteWebResource = client.resource(wurl + id+"/pod?pod="+podname); - ClientResponse deleteResponse = deleteWebResource.accept(MediaType.APPLICATION_JSON).put(ClientResponse.class); + WebResource changePodname = client.resource(wurl + id+"/pod?pod="+podname); + ClientResponse deleteResponse = changePodname.accept(MediaType.APPLICATION_JSON).put(ClientResponse.class); if (deleteResponse.getStatus() == ClientResponse.Status.OK.getStatusCode()) { LOGGER.info(String.format("Change podname for %s -> %s ", id, podname)); } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 3089f58405..f217d1b2d5 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -166,7 +166,7 @@ public boolean acceptElement(Element element) { }); if (rootPid != null) { try { - ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Reharvest from CDK Core","open", new ArrayList<>(Arrays.asList(rootPid.getTextContent().trim()))); + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", new ArrayList<>(Arrays.asList(rootPid.getTextContent().trim()))); this.reharvestManager.register(reharvestItem); } catch (DOMException e) { LOGGER.log(Level.SEVERE,e.getMessage(),e); From 8ed77e18a71b37b88b285678fa448ff933ffdc4a Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 1 May 2024 19:25:57 +0200 Subject: [PATCH 128/235] Version --- gradle.properties | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index 11d59301d0..ba0e9a9cb8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,4 +1,3 @@ -#org.gradle.java.home=c:\\Users\\happy\\Programs\\jdk18 group=cz.incad.kramerius -version=7.0.33_dev20_cdk +version=7.0.33_dev21_cdk From 60e3c4f2ce473e94c64c6e63a52dc2047714d0d1 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 07:45:39 +0200 Subject: [PATCH 129/235] Changed default filter --- .../rest/apiNew/client/v60/filter/DefaultFilter.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java index c436cd6044..be029d1c09 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java @@ -67,7 +67,7 @@ private String filter() { .collect(Collectors.toList()); // List eInsts = // libraries.enabledInstances().stream().map(OneInstance::getName)::getName).; - String enabled = eInsts.stream().collect(Collectors.joining(" OR ")); + String enabled = eInsts.stream().collect(Collectors.joining("+OR+")); return "cdk.collection:(" + enabled + ")"; } else { return null; @@ -77,7 +77,7 @@ private String filter() { @Override public String enhancedFilter(String f) { if (this.libraries.isAnyDisabled()) { - return f + " AND " + filter(); + return f + "+AND+" + filter(); } else return f; } From c5b1d242adc60916bf98e0efc1fbf06a72f1b5b9 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 07:48:39 +0200 Subject: [PATCH 130/235] Changed minor version --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index ba0e9a9cb8..cc8159b4ce 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=cz.incad.kramerius -version=7.0.33_dev21_cdk +version=7.0.33_dev22_cdk From f01950001ea2000ce977890db0d97b0fd959920b Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 09:18:55 +0200 Subject: [PATCH 131/235] Version changed --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index cc8159b4ce..921a056bf3 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=cz.incad.kramerius -version=7.0.33_dev22_cdk +version=7.0.33_dev24_cdk From 2444d0fd5d4c292e2bd1d8ada395d2a556a47156 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 10:22:24 +0200 Subject: [PATCH 132/235] DefaultFilter --- .../apiNew/client/v60/filter/DefaultFilter.java | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java index be029d1c09..d013b949a8 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java @@ -67,8 +67,13 @@ private String filter() { .collect(Collectors.toList()); // List eInsts = // libraries.enabledInstances().stream().map(OneInstance::getName)::getName).; - String enabled = eInsts.stream().collect(Collectors.joining("+OR+")); - return "cdk.collection:(" + enabled + ")"; + String enabled = eInsts.stream().collect(Collectors.joining(" OR ")); + try { + return "cdk.collection:(" + URLEncoder.encode(enabled,"UTF-8") + ")"; + } catch (UnsupportedEncodingException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + return "cdk.collection:(" + enabled + ")"; + } } else { return null; } @@ -77,7 +82,13 @@ private String filter() { @Override public String enhancedFilter(String f) { if (this.libraries.isAnyDisabled()) { - return f + "+AND+" + filter(); + try { + String retval = f + URLEncoder.encode(" AND ","UTF-8") + filter(); + return retval; + } catch (UnsupportedEncodingException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + return f + " AND " + filter(); + } } else return f; } From 1eb85b4257cc876ec5936e5d51d400213847c9eb Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 10:23:55 +0200 Subject: [PATCH 133/235] Version changed --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 921a056bf3..42f1d07781 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=cz.incad.kramerius -version=7.0.33_dev24_cdk +version=7.0.33_dev25_cdk From 5d81cb61e2f4e7903762118db1d173206170a591 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 10:50:18 +0200 Subject: [PATCH 134/235] Debug info message --- gradle.properties | 2 +- .../kramerius/rest/api/k5/client/search/SearchResource.java | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/gradle.properties b/gradle.properties index 42f1d07781..8158827af8 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=cz.incad.kramerius -version=7.0.33_dev25_cdk +version=7.0.33_dev26_cdk diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/search/SearchResource.java b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/search/SearchResource.java index c720c05068..2411a41051 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/search/SearchResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/search/SearchResource.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.Set; import java.util.Stack; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; @@ -96,6 +97,7 @@ public Response selectXML(@Context UriInfo uriInfo, @QueryParam("wt") String wt) } private String getEntityXML(UriInfo uriInfo) { + AtomicReference queryRef = new AtomicReference<>(); try { MultivaluedMap queryParameters = uriInfo .getQueryParameters(); @@ -112,6 +114,7 @@ private String getEntityXML(UriInfo uriInfo) { builder.append("&"); } } + queryRef.set(builder.toString()); InputStream istream = this.solrAccess.requestWithSelectReturningInputStream(builder.toString(), "xml"); ByteArrayOutputStream bos = new ByteArrayOutputStream(); @@ -128,7 +131,7 @@ private String getEntityXML(UriInfo uriInfo) { return strWriter.toString(); } catch (HttpResponseException e) { if (e.getStatusCode() == SC_BAD_REQUEST) { - LOGGER.log(Level.INFO, "SOLR Bad Request: " + uriInfo.getRequestUri()); + LOGGER.log(Level.SEVERE, String.format("Bad Request (api request = %s,\n solr request %s", uriInfo.getRequestUri(), queryRef.get())); throw new BadRequestException(e.getMessage()); } else { LOGGER.log(Level.INFO, e.getMessage(), e); From ed434398104603ff01a494e96e47531694ea0c3f Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 11:19:06 +0200 Subject: [PATCH 135/235] Debug message --- .../rest/api/k5/client/search/SearchResource.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/search/SearchResource.java b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/search/SearchResource.java index 2411a41051..6958342816 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/search/SearchResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/search/SearchResource.java @@ -131,7 +131,7 @@ private String getEntityXML(UriInfo uriInfo) { return strWriter.toString(); } catch (HttpResponseException e) { if (e.getStatusCode() == SC_BAD_REQUEST) { - LOGGER.log(Level.SEVERE, String.format("Bad Request (api request = %s,\n solr request %s", uriInfo.getRequestUri(), queryRef.get())); + LOGGER.log(Level.SEVERE, String.format("Bad Request (api request = %s,\n solr request %s)", uriInfo.getRequestUri(), queryRef.get())); throw new BadRequestException(e.getMessage()); } else { LOGGER.log(Level.INFO, e.getMessage(), e); @@ -195,6 +195,7 @@ public Response selectJSON(@Context UriInfo uriInfo, @QueryParam("wt") String wt } private String getEntityJSON(UriInfo uriInfo) { + AtomicReference queryRef = new AtomicReference<>(); try { MultivaluedMap queryParameters = uriInfo @@ -212,6 +213,8 @@ private String getEntityJSON(UriInfo uriInfo) { builder.append("&"); } } + queryRef.set(builder.toString()); + InputStream istream = this.solrAccess.requestWithSelectReturningInputStream(builder.toString(), "json"); ByteArrayOutputStream bos = new ByteArrayOutputStream(); @@ -225,7 +228,9 @@ private String getEntityJSON(UriInfo uriInfo) { return jsonObject.toString(); } catch (HttpResponseException e) { if (e.getStatusCode() == SC_BAD_REQUEST) { - LOGGER.log(Level.INFO, "SOLR Bad Request: " + uriInfo.getRequestUri()); + //LOGGER.log(Level.INFO, "SOLR Bad Request: " + uriInfo.getRequestUri()); + LOGGER.log(Level.SEVERE, String.format("Bad Request (api request = %s,\n solr request %s"), uriInfo.getRequestUri(), queryRef.get())); + throw new BadRequestException(e.getMessage()); } else { LOGGER.log(Level.INFO, e.getMessage(), e); From 8e598e619c8d959bc40f39608de0b0c2845021b1 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 11:20:33 +0200 Subject: [PATCH 136/235] Version changed --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 8158827af8..95efdfb7ec 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=cz.incad.kramerius -version=7.0.33_dev26_cdk +version=7.0.33_dev27_cdk From 40ff1dc750cbded5075b9d38059374f972e58985 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 11:25:42 +0200 Subject: [PATCH 137/235] Changed error message --- .../kramerius/rest/api/k5/client/search/SearchResource.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/search/SearchResource.java b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/search/SearchResource.java index 6958342816..e14e18f6a9 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/search/SearchResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/k5/client/search/SearchResource.java @@ -131,7 +131,8 @@ private String getEntityXML(UriInfo uriInfo) { return strWriter.toString(); } catch (HttpResponseException e) { if (e.getStatusCode() == SC_BAD_REQUEST) { - LOGGER.log(Level.SEVERE, String.format("Bad Request (api request = %s,\n solr request %s)", uriInfo.getRequestUri(), queryRef.get())); + String message = String.format("Bad Request (api request = %s,\n solr request %s)", uriInfo.getRequestUri(), queryRef.get()); + LOGGER.log(Level.SEVERE, message); throw new BadRequestException(e.getMessage()); } else { LOGGER.log(Level.INFO, e.getMessage(), e); @@ -229,7 +230,8 @@ private String getEntityJSON(UriInfo uriInfo) { } catch (HttpResponseException e) { if (e.getStatusCode() == SC_BAD_REQUEST) { //LOGGER.log(Level.INFO, "SOLR Bad Request: " + uriInfo.getRequestUri()); - LOGGER.log(Level.SEVERE, String.format("Bad Request (api request = %s,\n solr request %s"), uriInfo.getRequestUri(), queryRef.get())); + String message = String.format("Bad Request (api request = %s,\n solr request %s)", uriInfo.getRequestUri(), queryRef.get()); + LOGGER.log(Level.SEVERE, message); throw new BadRequestException(e.getMessage()); } else { From 0a5001b8962f1a6c7a9b2ea4cf49f11ab640972e Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 12:04:15 +0200 Subject: [PATCH 138/235] SearchResource - debug messages --- .../rest/apiNew/client/v60/SearchResource.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java index fb95fb8cd1..874ea690d9 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java @@ -48,6 +48,7 @@ import java.io.*; import java.net.URLEncoder; import java.util.*; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; @@ -95,8 +96,10 @@ public Response get(@Context UriInfo uriInfo, @QueryParam("wt") String wt) { } private String buildSearchResponseJson(UriInfo uriInfo) { + AtomicReference queryRef = new AtomicReference<>(); try { String solrQuery = buildSearchSolrQueryString(uriInfo); + queryRef.set(solrQuery); // filter String solrResponseJson = this.solrAccess.requestWithSelectReturningString(solrQuery, "json"); String uri = UriBuilder.fromResource(SearchResource.class).path("").build().toString(); @@ -104,7 +107,10 @@ private String buildSearchResponseJson(UriInfo uriInfo) { return jsonObject.toString(); } catch (HttpResponseException e) { if (e.getStatusCode() == SC_BAD_REQUEST) { - LOGGER.log(Level.INFO, "SOLR Bad Request: " + uriInfo.getRequestUri()); + + String message = String.format("Bad Request (api request = %s,\n solr request %s)", uriInfo.getRequestUri(), queryRef.get()); + LOGGER.log(Level.SEVERE, message); + throw new BadRequestException(e.getMessage()); } else { LOGGER.log(Level.INFO, e.getMessage(), e); @@ -120,8 +126,11 @@ private String buildSearchResponseJson(UriInfo uriInfo) { } private String buildSearchResponseXml(UriInfo uriInfo) { + AtomicReference queryRef = new AtomicReference<>(); try { String solrQuery = buildSearchSolrQueryString(uriInfo); + queryRef.set(solrQuery); + String solrResponseXml = this.solrAccess.requestWithSelectReturningString(solrQuery, "xml"); Document domObject = buildXmlFromRawSolrResponse(solrResponseXml); StringWriter strWriter = new StringWriter(); @@ -129,7 +138,11 @@ private String buildSearchResponseXml(UriInfo uriInfo) { return strWriter.toString(); } catch (HttpResponseException e) { if (e.getStatusCode() == SC_BAD_REQUEST) { - LOGGER.log(Level.INFO, "SOLR Bad Request: " + uriInfo.getRequestUri()); + //LOGGER.log(Level.INFO, "SOLR Bad Request: " + uriInfo.getRequestUri()); + + String message = String.format("Bad Request (api request = %s,\n solr request %s)", uriInfo.getRequestUri(), queryRef.get()); + LOGGER.log(Level.SEVERE, message); + throw new BadRequestException(e.getMessage()); } else { LOGGER.log(Level.INFO, e.getMessage(), e); From acdc05161b6809c823259aa747618bc8cb83b300 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 12:04:58 +0200 Subject: [PATCH 139/235] Debug version --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 95efdfb7ec..cb184a9939 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=cz.incad.kramerius -version=7.0.33_dev27_cdk +version=7.0.33_dev28_cdk From bc5f3d6fa0a69bacacebd678c08bb1385a20ac8f Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 12:23:02 +0200 Subject: [PATCH 140/235] Debug message - reasonPhrase --- gradle.properties | 2 +- .../kramerius/rest/apiNew/client/v60/SearchResource.java | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/gradle.properties b/gradle.properties index cb184a9939..0702fac163 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=cz.incad.kramerius -version=7.0.33_dev28_cdk +version=7.0.33_dev29_cdk diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java index 874ea690d9..c639752fd1 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java @@ -107,10 +107,12 @@ private String buildSearchResponseJson(UriInfo uriInfo) { return jsonObject.toString(); } catch (HttpResponseException e) { if (e.getStatusCode() == SC_BAD_REQUEST) { - + String reasonPhrase = e.getReasonPhrase(); + String message = String.format("Bad Request (api request = %s,\n solr request %s)", uriInfo.getRequestUri(), queryRef.get()); LOGGER.log(Level.SEVERE, message); - + LOGGER.log(Level.SEVERE, String.format("Reason phrase %s", reasonPhrase)); + throw new BadRequestException(e.getMessage()); } else { LOGGER.log(Level.INFO, e.getMessage(), e); From 838c67b320f147b730f6ce571da94176e57866e6 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 13:12:23 +0200 Subject: [PATCH 141/235] Error message from server --- .../cz/incad/kramerius/impl/SolrAccessImplNewIndex.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/shared/common/src/main/java/cz/incad/kramerius/impl/SolrAccessImplNewIndex.java b/shared/common/src/main/java/cz/incad/kramerius/impl/SolrAccessImplNewIndex.java index d9498a1b05..9376e1989b 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/impl/SolrAccessImplNewIndex.java +++ b/shared/common/src/main/java/cz/incad/kramerius/impl/SolrAccessImplNewIndex.java @@ -363,6 +363,13 @@ InputStream requestWithSelectReturningStream(String query, String type) throws I if (response.getStatusLine().getStatusCode() == SC_OK) { return readContentAndProvideThroughBufferedStream(response.getEntity()); } else { + + HttpEntity entity = response.getEntity(); + if (entity != null) { + InputStream stream = readContentAndProvideThroughBufferedStream(entity); + String strEntity = IOUtils.toString(stream, "UTF-8"); + LOGGER.log(Level.SEVERE, String.format("Error entity %s", strEntity)); + } throw new HttpResponseException(response.getStatusLine().getStatusCode(), response.getStatusLine().getReasonPhrase()); } } From 0208c8a6260345330efa3f3f56bf194e5081ec9c Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 13:13:08 +0200 Subject: [PATCH 142/235] New dev version --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 0702fac163..c41294c7eb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=cz.incad.kramerius -version=7.0.33_dev29_cdk +version=7.0.33_dev30_cdk From 7e27337126d4bf89f913ba99b23da5acd458ef4e Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 14:41:11 +0200 Subject: [PATCH 143/235] Version changed --- gradle.properties | 2 +- .../client/v60/filter/DefaultFilter.java | 19 ++++++------------- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/gradle.properties b/gradle.properties index c41294c7eb..3cd6c5c91e 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=cz.incad.kramerius -version=7.0.33_dev30_cdk +version=7.0.33_dev31_cdk diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java index d013b949a8..b452ba24dc 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/filter/DefaultFilter.java @@ -68,12 +68,9 @@ private String filter() { // List eInsts = // libraries.enabledInstances().stream().map(OneInstance::getName)::getName).; String enabled = eInsts.stream().collect(Collectors.joining(" OR ")); - try { - return "cdk.collection:(" + URLEncoder.encode(enabled,"UTF-8") + ")"; - } catch (UnsupportedEncodingException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - return "cdk.collection:(" + enabled + ")"; - } + String created = "cdk.collection:(" + enabled + ")"; + LOGGER.info(String.format("Created filter %s", enabled)); + return created; } else { return null; } @@ -82,13 +79,9 @@ private String filter() { @Override public String enhancedFilter(String f) { if (this.libraries.isAnyDisabled()) { - try { - String retval = f + URLEncoder.encode(" AND ","UTF-8") + filter(); - return retval; - } catch (UnsupportedEncodingException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - return f + " AND " + filter(); - } + String retval = f + " AND " + filter(); + LOGGER.info(String.format("Enhanced filter %s", retval)); + return retval; } else return f; } From 85ed7382491da88dbccba4fc950928ff7c518ee2 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 15:10:22 +0200 Subject: [PATCH 144/235] Changed OAI set --- rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java index d455e0fdc9..b0e754b653 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.io.StringReader; import java.io.StringWriter; +import java.net.URLEncoder; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -150,7 +151,7 @@ public void initToConfig(ConfigManager configManager) { public int numberOfDoc(ProxyFilter proxyFilter, SolrAccess solrAccess) throws IOException, ParserConfigurationException, SAXException { String query = String.format("q=%s&fl=pid&rows=%d&sort=pid+asc", this.filterQuery, 0); if (proxyFilter.newFilter() != null) { - query = query + String.format("&fq=%s", proxyFilter.newFilter()); + query = query + String.format("&fq=%s", URLEncoder.encode(proxyFilter.newFilter(), "UTF-8")); } String solrResponseXml = solrAccess.requestWithSelectReturningString(query, "xml"); Document document = XMLUtils.parseDocument(new StringReader(solrResponseXml)); From d12e7b4bc5bf4b903a53032027205f87ada1a9dd Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 2 May 2024 15:10:55 +0200 Subject: [PATCH 145/235] Changed version --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index 3cd6c5c91e..73d1fb5adb 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=cz.incad.kramerius -version=7.0.33_dev31_cdk +version=7.0.33_dev32_cdk From 2f4af25168d1c413d85c8e2a96fe04fb4439d04d Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 3 May 2024 18:43:13 +0200 Subject: [PATCH 146/235] Changed version --- gradle.properties | 2 +- .../kramerius/KubernetesReharvestProcess.java | 134 +++++++++++------- .../incad/kramerius/utils/ReharvestUtils.java | 5 +- .../cz/incad/kramerius/reharvest_v5.xml | 2 +- .../cz/incad/kramerius/reharvest_v7.xml | 2 +- .../v60/redirection/ProxyHandlerSupport.java | 67 +++++---- .../redirection/item/V5ForwardHandler.java | 8 +- .../redirection/item/V5RedirectHandler.java | 35 +++-- .../redirection/item/V7ForwardHandler.java | 9 +- 9 files changed, 154 insertions(+), 110 deletions(-) diff --git a/gradle.properties b/gradle.properties index 73d1fb5adb..d90b6f01c3 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=cz.incad.kramerius -version=7.0.33_dev32_cdk +version=7.0.33_dev33_cdk diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java index fc07169e3d..2b271522ad 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Map; import java.util.TimeZone; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Logger; import javax.ws.rs.PUT; @@ -36,7 +37,10 @@ public class KubernetesReharvestProcess { + public static final int DEFAULT_MAX_ITEMS_TO_DELETE = 10000; + public static final String ONLY_SHOW_CONFIGURATION = "ONLY_SHOW_CONFIGURATION"; + public static final String MAX_ITEMS_TO_DELETE = "MAX_ITEMS_TO_DELETE"; public static final Logger LOGGER = Logger.getLogger(KubernetesReharvestProcess.class.getName()); @@ -49,73 +53,104 @@ protected static Client buildClient() { } public static void main(String[] args) throws ParserConfigurationException, IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchMethodException, MigrateSolrIndexException, IOException, SAXException { - + TimeZone.setDefault(TimeZone.getTimeZone("Europe/Prague")); Map env = System.getenv(); Map iterationMap = KubernetesEnvSupport.iterationMap(env); + if (!iterationMap.containsKey("batch")) { + iterationMap.put("batch", "45"); + } Map reharvestMap = KubernetesEnvSupport.reharvestMap(env); Map destinationMap = KubernetesEnvSupport.destinationMap(env); Map proxyMap = KubernetesEnvSupport.proxyMap(env); boolean onlyShowConfiguration = env.containsKey(ONLY_SHOW_CONFIGURATION); + int maxItemsToDelete = env.containsKey(MAX_ITEMS_TO_DELETE) ? Integer.parseInt(env.get(MAX_ITEMS_TO_DELETE)) : DEFAULT_MAX_ITEMS_TO_DELETE; + AtomicReference idReference = new AtomicReference<>(); - if (reharvestMap.containsKey("url") && proxyMap.containsKey("url")) { - Client client = buildClient(); - String wurl = reharvestMap.get("url"); - if (!wurl.endsWith("/")) { - wurl = wurl + "/"; - } + try { - // Top item - WebResource topWebResource = client.resource(wurl + "top?state=open"); - ClientResponse topItemFrom = topWebResource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); - if (topItemFrom.getStatus() == ClientResponse.Status.OK.getStatusCode()) { - - String t = topItemFrom.getEntity(String.class); - - JSONObject itemObject = new JSONObject(t); - String id = itemObject.getString("id"); - JSONArray pids = itemObject.getJSONArray("pids"); - - if (!onlyShowConfiguration) { - changeState(client, wurl, id,"running"); - String podname = env.get("HOSTNAME"); - if (cz.incad.kramerius.utils.StringUtils.isAnyString(podname)) { - changePodname(client, wurl, id, podname); - } + + if (reharvestMap.containsKey("url") && proxyMap.containsKey("url")) { + Client client = buildClient(); + String wurl = reharvestMap.get("url"); + if (!wurl.endsWith("/")) { + wurl = wurl + "/"; } + + // Top item + WebResource topWebResource = client.resource(wurl + "top?state=open"); + ClientResponse topItemFrom = topWebResource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); + if (topItemFrom.getStatus() == ClientResponse.Status.OK.getStatusCode()) { - // find all pids - List> allPidsList = ReharvestUtils.findAllPidsByGivenRootPid(iterationMap, client, pids); - // delete all pids - ReharvestUtils.deleteAllGivenPids(client, destinationMap, allPidsList, onlyShowConfiguration); + String t = topItemFrom.getEntity(String.class); - // List acronyms = new ArrayList(); - // https://api.val.ceskadigitalniknihovna.cz/search/api/admin/v7.0/connected/ - String proxyURl = proxyMap.get("url"); - if (proxyURl != null) { - Map configurations = libraryConfigurations(client, proxyURl); + JSONObject itemObject = new JSONObject(t); + String id = itemObject.getString("id"); + idReference.set(id); - for (int i = 0; i < pids.length(); i++) { - String p = pids.getString(i); - // reharvesting - ReharvestUtils.reharvestPIDFromGivenCollections(p, configurations, ""+onlyShowConfiguration, destinationMap); - } - + JSONArray pids = itemObject.getJSONArray("pids"); + if (!onlyShowConfiguration) { - changeState(client, wurl, id,"closed"); + changeState(client, wurl, id,"running"); + String podname = env.get("HOSTNAME"); + if (cz.incad.kramerius.utils.StringUtils.isAnyString(podname)) { + changePodname(client, wurl, id, podname); + } + } + + // find all pids + List> allPidsList = ReharvestUtils.findAllPidsByGivenRootPid(iterationMap, client, pids); + // check size; if size > 10000 - fail state + if (allPidsList.size() < maxItemsToDelete) { + // delete all pids + ReharvestUtils.deleteAllGivenPids(client, destinationMap, allPidsList, onlyShowConfiguration); + + // List acronyms = new ArrayList(); + // https://api.val.ceskadigitalniknihovna.cz/search/api/admin/v7.0/connected/ + String proxyURl = proxyMap.get("url"); + if (proxyURl != null) { + Map configurations = libraryConfigurations(client, proxyURl); + + + for (int i = 0; i < pids.length(); i++) { + String p = pids.getString(i); + // reharvesting + ReharvestUtils.reharvestPIDFromGivenCollections(p, configurations, ""+onlyShowConfiguration, destinationMap, iterationMap); + } + + if (!onlyShowConfiguration) { + changeState(client, wurl, id,"closed"); + } + } else { + LOGGER.severe("No proxy configuration"); + } + } else { + changeState(client, wurl, id,"too_big"); + String compare = String.format("delete.size() %d >= maxItemstoDelete %d", allPidsList.size(), maxItemsToDelete); + LOGGER.severe(String.format( "Too big to reharvest (%s) -> manual change ", compare)); } - } else { - LOGGER.severe("No proxy configuration"); + } else if (topItemFrom.getStatus() == ClientResponse.Status.NOT_FOUND.getStatusCode()) { + LOGGER.info("No item to harvest"); } - } else if (topItemFrom.getStatus() == ClientResponse.Status.NOT_FOUND.getStatusCode()) { - LOGGER.info("No item to harvest"); + } else { + LOGGER.severe("No proxy or reharvest configuration"); } - } else { - LOGGER.severe("No proxy or reharvest configuration"); + + } catch (Throwable thr) { + if (idReference.get() != null) { + Client client = buildClient(); + String wurl = reharvestMap.get("url"); + if (!wurl.endsWith("/")) { + wurl = wurl + "/"; + } + changeState(client, wurl, idReference.get(),"failed"); + } + LOGGER.severe("Exception ex" + thr.getMessage()); } + } private static void changeState(Client client, String wurl, String id, String changeState) { @@ -127,11 +162,6 @@ private static void changeState(Client client, String wurl, String id, String ch } } -// @PUT -// @Path("{id}/pod") -// @Produces(MediaType.APPLICATION_JSON) -// public Response changePod(@PathParam("id") String id, @QueryParam("pod") String pod) { - private static void changePodname(Client client, String wurl, String id, String podname) { //String changeState = "closed"; WebResource changePodname = client.resource(wurl + id+"/pod?pod="+podname); @@ -152,6 +182,7 @@ public static Map libraryConfigurations(Client client, Strin for (Object key : responseAllConnectedObject.keySet()) { JSONObject lib = responseAllConnectedObject.getJSONObject(key.toString()); if (lib.has("status") && lib.getBoolean("status")) { + String configURl = proxyURl; if (!configURl.endsWith("/")) { configURl += "/"; @@ -162,6 +193,7 @@ public static Map libraryConfigurations(Client client, Strin ClientResponse configReourceStatus = configResource.accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); if (configReourceStatus.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + configurations.put(key.toString(), new JSONObject(configReourceStatus.getEntity(String.class))); } diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index 85de687754..2c4884e113 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -173,7 +173,7 @@ public static String fq(String api, String pid) { return "root_pid:\""+pid+"\""; } - public static void reharvestPIDFromGivenCollections(String pid, Map collectionConfigurations, String onlyShowConfiguration, Map destinationMap) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchMethodException, MigrateSolrIndexException, IOException, ParserConfigurationException, SAXException { + public static void reharvestPIDFromGivenCollections(String pid, Map collectionConfigurations, String onlyShowConfiguration, Map destinationMap, Map iterationMap) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchMethodException, MigrateSolrIndexException, IOException, ParserConfigurationException, SAXException { List harvestFiles = new ArrayList<>(); for (String ac : collectionConfigurations.keySet()) { try { @@ -186,7 +186,7 @@ public static void reharvestPIDFromGivenCollections(String pid, Map iteration = new HashMap<>(); + Map iteration = new HashMap<>(iterationMap); //http://mzk-tunnel.cdk-proxy.svc.cluster.local/search" //http://knav-tunnel.cdk-proxy.svc.cluster.local/search/api/v5.0/cdk/forward/sync/solr //v7.0 @@ -199,7 +199,6 @@ public static void reharvestPIDFromGivenCollections(String pid, Mapselect - 80 + $iteration.batch$ $iteration.url$ diff --git a/processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_v7.xml b/processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_v7.xml index ac23cd2887..ad895406fd 100644 --- a/processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_v7.xml +++ b/processes/cdkreharvest/src/main/resources/cz/incad/kramerius/reharvest_v7.xml @@ -25,7 +25,7 @@ $iteration.url$ select - 80 + $iteration.batch$ pid diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index f217d1b2d5..b1e8178878 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -151,42 +151,47 @@ public void write(OutputStream output) throws IOException, WebApplicationExcepti } else { // event for reharvest if (response.getStatus() == 404) { - if (reharvestManager != null && pid != null) { - try { - Document solrDataByPid = this.solrAccess.getSolrDataByPid(pid); - Element rootPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - if (element.getNodeName().equals("str")) { - String fieldName = element.getAttribute("name"); - return fieldName.equals("root.pid"); - } - return false; - } - }); - if (rootPid != null) { - try { - ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", new ArrayList<>(Arrays.asList(rootPid.getTextContent().trim()))); - this.reharvestManager.register(reharvestItem); - } catch (DOMException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } catch (AlreadyRegistedPidsException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); - } - } else { - LOGGER.log(Level.SEVERE, "Cannot find root.pid element"); - } - } catch (IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage()); - } - } else { - LOGGER.log(Level.SEVERE,"No reharvest manager or pid "); - } + deleteTriggeToReharvest(pid); } return Response.status(response.getStatus()).build(); } } + protected void deleteTriggeToReharvest(String pid) { + if (reharvestManager != null && pid != null) { + try { + Document solrDataByPid = this.solrAccess.getSolrDataByPid(pid); + Element rootPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("str")) { + String fieldName = element.getAttribute("name"); + return fieldName.equals("root.pid"); + } + return false; + } + }); + if (rootPid != null) { + try { + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", new ArrayList<>(Arrays.asList(rootPid.getTextContent().trim()))); + reharvestItem.setState("waiting_for_approve"); + this.reharvestManager.register(reharvestItem); + } catch (DOMException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } catch (AlreadyRegistedPidsException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } + } else { + LOGGER.log(Level.SEVERE, "Cannot find root.pid element"); + } + } catch (IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage()); + } + } else { + LOGGER.log(Level.SEVERE,"No reharvest manager or pid "); + } + } + protected void mockSession() { if (!user.getSessionAttributes().containsKey("shib-session-id")) { this.user.addSessionAttribute("shib-session-id", "_dd68cbd66641c9b647b05509ac0241fa"); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java index 72907cd56c..1fce92af29 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java @@ -57,7 +57,7 @@ public Response image(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, this.pid); + return buildForwardResponseGET(url, null); } } @@ -100,7 +100,7 @@ public Response zoomifyImageProperties(RequestMethodName method) throws ProxyHan if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, this.pid); + return buildForwardResponseGET(url, null); } } @@ -109,7 +109,7 @@ public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHan String baseurl = this.forwardUrl(); String formatted = String.format("api/v5.0/cdk/forward/zoomify/%s/%s/%s", this.pid, tileGroupStr, tileStr); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + formatted; - return buildForwardResponseGET(url, this.pid); + return buildForwardResponseGET(url, null); } @Override @@ -192,7 +192,7 @@ public Response iiifTile(RequestMethodName method, String pid, String region, St if (splited.length > 1) { mime = IIIF_SUPPORTED_MIMETYPES.containsKey(splited[1]) ? IIIF_SUPPORTED_MIMETYPES.get(splited[1]) : defaultMime; } - return buildForwardResponseGET(url, mime); + return buildForwardResponseGET(url, null, mime); } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java index 30df4c0539..8dad1db896 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java @@ -179,22 +179,27 @@ public Response providedByLicenses() throws ProxyHandlerException { @Override public Response info() throws ProxyHandlerException { try { - JSONObject streams = new JSONObject(retrieveStreams()); - JSONObject info = new JSONObject(retrieveInfo()); JSONObject basicDoc = retrieveBasicDoc(this.pid); - - JSONObject data = extractAvailableDataInfo(streams); - JSONObject image = extractImageSourceInfo(info, streams, basicDoc); - JSONObject struct = extractStructureInfo(this.source, info, basicDoc); - - JSONObject json = new JSONObject(); - json.put("data", data); - json.put("structure", struct); - json.put("image", image); - // TODO: providedByLicenses - json.put("providedByLicenses", providedByLicense()); - - return Response.ok(json).build(); + if (basicDoc != null) { + JSONObject streams = new JSONObject(retrieveStreams()); + JSONObject info = new JSONObject(retrieveInfo()); + JSONObject data = extractAvailableDataInfo(streams); + JSONObject image = extractImageSourceInfo(info, streams, basicDoc); + JSONObject struct = extractStructureInfo(this.source, info, basicDoc); + + JSONObject json = new JSONObject(); + json.put("data", data); + json.put("structure", struct); + json.put("image", image); + // TODO: providedByLicenses + json.put("providedByLicenses", providedByLicense()); + return Response.ok(json).build(); + } else { + this.deleteTriggeToReharvest(this.pid); + return Response.status(Response.Status.NOT_FOUND).build(); + } + + } catch (JSONException | LexerException | IOException | RepositoryException e) { throw new ProxyHandlerException(e); } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index 54d66f9d12..e81b4bf13c 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -82,6 +82,9 @@ public Response info() throws ProxyHandlerException { return respEntity.build(); } else { + if (response.getStatus() == 404) { + this.deleteTriggeToReharvest(this.pid); + } return Response.status(response.getStatus()).build(); } @@ -96,7 +99,7 @@ public Response image(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, this.pid); + return buildForwardResponseGET(url, null); } } @@ -134,7 +137,7 @@ public Response zoomifyImageProperties(RequestMethodName method) throws ProxyHan String baseurl = forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/zoomify/" + this.pid + "/ImageProperties.xml"; - return buildForwardResponseGET(url, this.pid); + return buildForwardResponseGET(url, null); } } @@ -145,7 +148,7 @@ public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHan String baseurl = forwardUrl(); String formatted = String.format("api/cdk/v7.0/forward/zoomify/%s/%s/%s", this.pid, tileGroupStr, tileStr); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + formatted; - return buildForwardResponseGET(url, this.pid); + return buildForwardResponseGET(url, null); } From e0a224762269667066f0a62941ed3165aea5a7e3 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 3 May 2024 20:46:33 +0200 Subject: [PATCH 147/235] Reharvests - registering --- .../admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java index 81c2c49f9f..141aa39633 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java @@ -21,7 +21,7 @@ public MemoryReharvestManagerImpl() { } public void register(ReharvestItem item) throws AlreadyRegistedPidsException { - List alreadyRegistredPids = this.items.stream().filter(x -> x.getState() != null && x.getState().equals("open")).map(ReharvestItem::getPids).flatMap(List::stream).collect(Collectors.toList()); + List alreadyRegistredPids = this.items.stream().filter(x -> x.getState() != null && (x.getState().equals("open") || x.getState().startsWith("waiting_for"))).map(ReharvestItem::getPids).flatMap(List::stream).collect(Collectors.toList()); List intersection = new ArrayList<>(item.getPids()); intersection.retainAll(alreadyRegistredPids); From 72105584c32c0fd32fd345734b4d76458aec2249 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 3 May 2024 21:28:01 +0200 Subject: [PATCH 148/235] Version changed --- gradle.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle.properties b/gradle.properties index d90b6f01c3..587d9989d4 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,3 +1,3 @@ group=cz.incad.kramerius -version=7.0.33_dev33_cdk +version=7.0.33_dev34_cdk From 7a3745a2849e7e363f40f144e086e501eba498c5 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 16 May 2024 05:35:36 +0200 Subject: [PATCH 149/235] OAISet --- rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java index b0e754b653..2ae8740d14 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java @@ -174,7 +174,8 @@ public OAIResults findRecords(ProxyFilter proxyFilter, SolrAccess solrAccess,Str String fq = proxyFilter.newFilter(); String query = String.format("q=%s&cursorMark=%s&fl=pid+cdk.leaders+cdk.collection+cdk.leader&rows=%d&sort=compositeId+asc", this.filterQuery, cursor, rows); if (fq != null) { - query = query + String.format("&fq=%s", fq); + String encodedFq = URLEncoder.encode(fq, "UTF-8"); + query = query + String.format("&fq=%s", encodedFq); } String solrResponseXml = solrAccess.requestWithSelectReturningString(query, "xml"); From bad1c69a550e6d8b23010a51261e5f9bdfd2c594 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 27 Jun 2024 08:10:08 +0200 Subject: [PATCH 150/235] Reharvests --- processes/cdkreharvest/Dockerfile | 10 + processes/cdkreharvest/bin/.gitignore | 1 + processes/cdkreharvest/build.bat | 2 + processes/cdkreharvest/kibana.log.1 | 0 processes/cdkreharvest/kibana.log.1.lck | 0 processes/cdkreharvest/push.bat | 2 + .../kramerius/KubernetesReharvestProcess.java | 25 +-- .../incad/kramerius/utils/ReharvestUtils.java | 129 ++++++++----- .../rest/api/guice/ApiServletModule.java | 3 +- .../admin/v10/reharvest/ReharvestItem.java | 122 ++++++++---- .../v10/reharvest/ReharvestResource.java | 58 +++++- .../impl/MemoryReharvestManagerImpl.java | 5 +- .../impl/SolrReharvestManagerImpl.java | 178 ++++++++++++++++++ .../v60/redirection/ProxyHandlerSupport.java | 31 ++- .../cz/incad/kramerius/solr/SolrModule.java | 8 + .../kramerius/utils/conf/KConfiguration.java | 4 + 16 files changed, 461 insertions(+), 117 deletions(-) create mode 100644 processes/cdkreharvest/Dockerfile create mode 100644 processes/cdkreharvest/bin/.gitignore create mode 100644 processes/cdkreharvest/build.bat create mode 100644 processes/cdkreharvest/kibana.log.1 create mode 100644 processes/cdkreharvest/kibana.log.1.lck create mode 100644 processes/cdkreharvest/push.bat create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java diff --git a/processes/cdkreharvest/Dockerfile b/processes/cdkreharvest/Dockerfile new file mode 100644 index 0000000000..f81ac29b5f --- /dev/null +++ b/processes/cdkreharvest/Dockerfile @@ -0,0 +1,10 @@ +#FROM openjdk:11-jre +#ADD build/libs/process-7.0.2-beta1_cdk.jar app.jar +#ENTRYPOINT ["java","-jar","/app.jar"] +FROM openjdk:11-jre +LABEL maintainer=pavel.stastny +WORKDIR /app +COPY build/install/cdkreharvest/lib lib/ +COPY build/install/cdkreharvest/bin bin/ +ENTRYPOINT ["bin/cdkreharvest"] + diff --git a/processes/cdkreharvest/bin/.gitignore b/processes/cdkreharvest/bin/.gitignore new file mode 100644 index 0000000000..ddf9c65631 --- /dev/null +++ b/processes/cdkreharvest/bin/.gitignore @@ -0,0 +1 @@ +/main/ diff --git a/processes/cdkreharvest/build.bat b/processes/cdkreharvest/build.bat new file mode 100644 index 0000000000..469fdd006d --- /dev/null +++ b/processes/cdkreharvest/build.bat @@ -0,0 +1,2 @@ +@echo off +docker build -t pavelstastny/cdkreharvest:7.0.33-reharvest-dev12 . diff --git a/processes/cdkreharvest/kibana.log.1 b/processes/cdkreharvest/kibana.log.1 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/processes/cdkreharvest/kibana.log.1.lck b/processes/cdkreharvest/kibana.log.1.lck new file mode 100644 index 0000000000..e69de29bb2 diff --git a/processes/cdkreharvest/push.bat b/processes/cdkreharvest/push.bat new file mode 100644 index 0000000000..48765a50b9 --- /dev/null +++ b/processes/cdkreharvest/push.bat @@ -0,0 +1,2 @@ +@echo off +docker push pavelstastny/cdkreharvest:7.0.33-reharvest-dev12 diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java index 2b271522ad..5c3da9b842 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -31,6 +31,7 @@ import com.sun.jersey.api.json.JSONConfiguration; import antlr.StringUtils; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; import cz.incad.kramerius.service.MigrateSolrIndexException; import cz.incad.kramerius.services.utils.kubernetes.KubernetesEnvSupport; import cz.incad.kramerius.utils.ReharvestUtils; @@ -68,10 +69,8 @@ public static void main(String[] args) throws ParserConfigurationException, Ille int maxItemsToDelete = env.containsKey(MAX_ITEMS_TO_DELETE) ? Integer.parseInt(env.get(MAX_ITEMS_TO_DELETE)) : DEFAULT_MAX_ITEMS_TO_DELETE; AtomicReference idReference = new AtomicReference<>(); - try { - - + if (reharvestMap.containsKey("url") && proxyMap.containsKey("url")) { Client client = buildClient(); String wurl = reharvestMap.get("url"); @@ -83,16 +82,11 @@ public static void main(String[] args) throws ParserConfigurationException, Ille WebResource topWebResource = client.resource(wurl + "top?state=open"); ClientResponse topItemFrom = topWebResource.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class); if (topItemFrom.getStatus() == ClientResponse.Status.OK.getStatusCode()) { - String t = topItemFrom.getEntity(String.class); - JSONObject itemObject = new JSONObject(t); - String id = itemObject.getString("id"); idReference.set(id); - - JSONArray pids = itemObject.getJSONArray("pids"); - + String pid = itemObject.getString("pid"); if (!onlyShowConfiguration) { changeState(client, wurl, id,"running"); String podname = env.get("HOSTNAME"); @@ -102,7 +96,7 @@ public static void main(String[] args) throws ParserConfigurationException, Ille } // find all pids - List> allPidsList = ReharvestUtils.findAllPidsByGivenRootPid(iterationMap, client, pids); + List> allPidsList = ReharvestUtils.findPidByType(iterationMap, client, ReharvestItem.fromJSON(itemObject)); // check size; if size > 10000 - fail state if (allPidsList.size() < maxItemsToDelete) { // delete all pids @@ -113,14 +107,9 @@ public static void main(String[] args) throws ParserConfigurationException, Ille String proxyURl = proxyMap.get("url"); if (proxyURl != null) { Map configurations = libraryConfigurations(client, proxyURl); - - - for (int i = 0; i < pids.length(); i++) { - String p = pids.getString(i); - // reharvesting - ReharvestUtils.reharvestPIDFromGivenCollections(p, configurations, ""+onlyShowConfiguration, destinationMap, iterationMap); - } - + + ReharvestUtils.reharvestPIDFromGivenCollections(pid, configurations, ""+onlyShowConfiguration, destinationMap, iterationMap); + if (!onlyShowConfiguration) { changeState(client, wurl, id,"closed"); } diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index 2c4884e113..ce8822c535 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -10,6 +10,8 @@ import java.io.IOException; import java.io.InputStream; import java.io.StringWriter; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashSet; @@ -36,6 +38,7 @@ import com.sun.jersey.api.client.Client; import cz.incad.kramerius.KubernetesReharvestProcess; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; import cz.incad.kramerius.service.MigrateSolrIndexException; import cz.incad.kramerius.services.ParallelProcessImpl; import cz.incad.kramerius.services.utils.SolrUtils; @@ -83,66 +86,74 @@ public static void deleteAllGivenPids(Client client,Map destinat } /** Find all pids by given root.pid */ - public static List> findAllPidsByGivenRootPid(Map iterationMap, Client client, JSONArray pids) { + public static List> findPidByType(Map iterationMap, Client client, ReharvestItem item) { LinkedHashSet> allPids = new LinkedHashSet<>(); - for (int i = 0; i < pids.length(); i++) { - String pid = pids.getString(i); - //allPids.add(pid); - String iterationUrl = iterationMap.get("url"); - - String masterQuery = "*:*"; - String filterQuery = "root.pid:\"" + pid + "\""; - try { - String cursorMark = null; - String queryCursorMark = null; - do { + String iterationUrl = iterationMap.get("url"); + + String masterQuery = "*:*"; + String filterQuery = "none"; + switch(item.getTypeOfReharvest()) { + case root: + filterQuery = "root.pid:\"" + item.getRootPid() + "\""; + break; + case children: + String ownPidPath = item.getOwnPidPath(); + ownPidPath = ownPidPath.replaceAll(":", "\\\\:")+"/*"; + filterQuery = String.format("own_pid_path:%s", ownPidPath);; + break; + } + + + try { + String cursorMark = null; + String queryCursorMark = null; + do { + + Element element = pidsCursorQuery(client, iterationUrl, masterQuery, cursorMark, 3000, + filterQuery, "select", "compositeId+pid", "compositeId asc", "", ""); + cursorMark = findCursorMark(element); + queryCursorMark = findQueryCursorMark(element); + + List docs = XMLUtils.getElementsRecursive(element, new XMLUtils.ElementsFilter() { - Element element = pidsCursorQuery(client, iterationUrl, masterQuery, cursorMark, 3000, - filterQuery, "select", "compositeId+pid", "compositeId asc", "", ""); - cursorMark = findCursorMark(element); - queryCursorMark = findQueryCursorMark(element); + @Override + public boolean acceptElement(Element element) { + return element.getNodeName().equals("doc"); + } + }); + + List> pairs = docs.stream().map(doc-> { - List docs = XMLUtils.getElementsRecursive(element, new XMLUtils.ElementsFilter() { + Element pidElm = XMLUtils.findElement(doc, new XMLUtils.ElementsFilter() { @Override public boolean acceptElement(Element element) { - return element.getNodeName().equals("doc"); + String name = element.getAttribute("name"); + return (name != null && name.equals("pid")); + } + }); + + Element compositeIdElm = XMLUtils.findElement(doc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + return (name != null && name.equals("compositeId")); } }); - List> pairs = docs.stream().map(doc-> { - - Element pidElm = XMLUtils.findElement(doc, new XMLUtils.ElementsFilter() { - - @Override - public boolean acceptElement(Element element) { - String name = element.getAttribute("name"); - return (name != null && name.equals("pid")); - } - }); - - Element compositeIdElm = XMLUtils.findElement(doc, new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - String name = element.getAttribute("name"); - return (name != null && name.equals("compositeId")); - } - }); - - - if (pidElm != null && compositeIdElm != null ) { - return Pair.of(pidElm.getTextContent(), compositeIdElm.getTextContent()); - } else return null; - }).filter(x -> x != null).collect(Collectors.toList()); - - allPids.addAll(pairs); - LOGGER.info(String.format( "Collected pids to delete %d", allPids.size())); - } while ((cursorMark != null && queryCursorMark != null) - && !cursorMark.equals(queryCursorMark)); - } catch (ParserConfigurationException | SAXException | IOException e) { - throw new RuntimeException(e); - } + if (pidElm != null && compositeIdElm != null ) { + return Pair.of(pidElm.getTextContent(), compositeIdElm.getTextContent()); + } else return null; + }).filter(x -> x != null).collect(Collectors.toList()); + + allPids.addAll(pairs); + LOGGER.info(String.format( "Collected pids to delete %d", allPids.size())); + + } while ((cursorMark != null && queryCursorMark != null) + && !cursorMark.equals(queryCursorMark)); + } catch (ParserConfigurationException | SAXException | IOException e) { + throw new RuntimeException(e); } return new ArrayList>(allPids); } @@ -241,4 +252,22 @@ public static boolean isOnlyShowConfiguration(String onlyShowConfiguration) { } +// public static void main(String[] args) throws UnsupportedEncodingException { +// +// Client client = Client.create(); +// Map iterationMap = new HashMap<>(); +// iterationMap.put("url", "https://api.ceskadigitalniknihovna.cz/search/api/client/v7.0/"); +// JSONArray array = new JSONArray(); +// array.put("uuid:d0ca97a4-d9a3-4c9c-b171-d0625c2d3a7c"); +// findAllPidsByGivenRootPid(iterationMap, client, array, null); +// +// client.resource("https://api.ceskadigitalniknihovna.cz/search/api/client/v7.0/search?q=*:*") +// String ownPidPath = "uuid\\:661bcc57-349e-47d8-a708-32fb8a93e793/*"; +// +// String fq = "root.pid:\"" + "uuid:xxxxx" + "\""; +// String encode = URLEncoder.encode(fq,"UTF-8"); +// System.out.println(encode); +// +// } + } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java index 99903f8045..ec3f1873d4 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/api/guice/ApiServletModule.java @@ -35,6 +35,7 @@ import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestResource; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.impl.MemoryReharvestManagerImpl; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.impl.SolrReharvestManagerImpl; import cz.incad.kramerius.rest.apiNew.admin.v10.rights.RightsResource; import cz.incad.kramerius.rest.api.k5.admin.statistics.StatisticsResource; import cz.incad.kramerius.rest.apiNew.admin.v10.rights.RolesResource; @@ -149,7 +150,7 @@ protected void configureServlets() { // cdk bind(TimestampStore.class).to(SolrTimestampStore.class).asEagerSingleton(); bind(Instances.class).to(DefaultPropertiesInstances.class).asEagerSingleton(); - bind(ReharvestManager.class).to(MemoryReharvestManagerImpl.class).asEagerSingleton(); + bind(ReharvestManager.class).to(SolrReharvestManagerImpl.class).asEagerSingleton(); bind(ProxyFilter.class).to(DefaultFilter.class); bind(ConnectedInfoResource.class); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java index 350d8866c6..c9b6fa0b85 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java @@ -20,39 +20,51 @@ */ public class ReharvestItem { + public static enum TypeOfReharvset { + root, children; + } + // static enum ReharvestItemState { // open, closed; // } - - //public static FastDateFormat FORMAT = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.sss'Z'", TimeZone.getTimeZone("UTC")); +// public static FastDateFormat FORMAT = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.sss'Z'", TimeZone.getTimeZone("UTC")); - private static final String TIMESTAMP_KEYWORD = "timestamp"; - private static final String PIDS_KEYWORD = "pids"; - private static final String TYPE_KEYWORD = "type"; - private static final String STATE_KEYWORD = "state"; - private static final String NAME_KEYWORD = "name"; - private static final String ID_KEYWORD = "id"; - private static final String POD_NAME_KEYWORD = "pod"; + public static final String TIMESTAMP_KEYWORD = "indexed"; + public static final String PID_KEYWORD = "pid"; + public static final String OWN_PID_PATH = "own_pid_path"; + public static final String ROOT_PID = "root.pid"; + public static final String TYPE_KEYWORD = "type"; + + public static final String STATE_KEYWORD = "state"; + public static final String NAME_KEYWORD = "name"; + public static final String ID_KEYWORD = "id"; + public static final String POD_NAME_KEYWORD = "pod"; // unique identifier private String id; private String name; private String state; - private String type; - private List pids= new ArrayList<>(); + //private String type; + private String pid; + private String rootPid; + private String ownPidPath; + private TypeOfReharvset typeOfReharvest = TypeOfReharvset.root; + + private Instant timestamp = Instant.now(); private String podname; private List libraries = new ArrayList<>(); - public ReharvestItem(String id, String name, String state, List pids) { + public ReharvestItem(String id, String name, String state, String pid, String ownPidPath) { super(); this.id = id; this.name = name; this.state = state; - this.pids = pids; + this.pid = pid; + this.ownPidPath = ownPidPath; } public ReharvestItem(String id) { @@ -80,12 +92,13 @@ public void setState(String state) { this.state = state; } - public List getPids() { - return pids; + public String getPid() { + return this.pid; } - public void setPids(List pids) { - this.pids = pids; + + public void setPid(String pid) { + this.pid = pid; } public List getLibraries() { @@ -104,14 +117,6 @@ public void setTimestamp(Instant timestamp) { this.timestamp = timestamp; } - public String getType() { - return type; - } - - public void setType(String type) { - this.type = type; - } - public void setPodname(String podname) { this.podname = podname; @@ -121,6 +126,32 @@ public String getPodname() { return podname; } + public String getOwnPidPath() { + return ownPidPath; + } + + public void setOwnPidPath(String ownPidPath) { + this.ownPidPath = ownPidPath; + } + + public TypeOfReharvset getTypeOfReharvest() { + return typeOfReharvest; + } + + + public void setTypeOfReharvest(TypeOfReharvset typeOfReharvest) { + this.typeOfReharvest = typeOfReharvest; + } + + + public String getRootPid() { + return rootPid; + } + + public void setRootPid(String rootPid) { + this.rootPid = rootPid; + } + public JSONObject toJSON() { JSONObject obj = new JSONObject(); @@ -129,13 +160,7 @@ public JSONObject toJSON() { if (this.state != null) { obj.put(STATE_KEYWORD, this.state); } - if (this.type != null) { - obj.put(TYPE_KEYWORD, this.type); - } - - JSONArray jsonArray = new JSONArray(); - this.pids.forEach(jsonArray::put); - obj.put(PIDS_KEYWORD, jsonArray); + obj.put(PID_KEYWORD, this.pid); if (!this.libraries.isEmpty()) { JSONArray libsArray = new JSONArray(); @@ -146,6 +171,15 @@ public JSONObject toJSON() { obj.put(POD_NAME_KEYWORD, this.podname); } + if (this.ownPidPath != null) { + obj.put(OWN_PID_PATH, this.ownPidPath); + } + + if (this.rootPid != null) { + obj.put(ROOT_PID, this.rootPid); + } + + obj.put(TYPE_KEYWORD, this.typeOfReharvest.name()); obj.put(TIMESTAMP_KEYWORD,DateTimeFormatter.ISO_INSTANT.format(this.timestamp)); return obj; @@ -154,20 +188,21 @@ public JSONObject toJSON() { public static ReharvestItem fromJSON(JSONObject json) throws ParseException { String id = json.getString(ID_KEYWORD); String name= json.getString(NAME_KEYWORD); - JSONArray array= json.getJSONArray(PIDS_KEYWORD); - List pids = new ArrayList<>(); - for (int i = 0; i < array.length(); i++) { - pids.add(array.getString(i)); - } - + String pid = json.getString(PID_KEYWORD); + String ownPidPath = json.optString(OWN_PID_PATH); + String rootPid = json.optString(ROOT_PID); + String state = json.optString(STATE_KEYWORD); - String type = json.optString(TYPE_KEYWORD); + //String type = json.optString(TYPE_KEYWORD); ReharvestItem item = new ReharvestItem(id); item.setName(name); - item.setPids(pids); + item.setPid(pid); + item.setOwnPidPath(ownPidPath); + item.setRootPid(rootPid); + item.setState(StringUtils.isAnyString(state) ? state : "open"); - item.setType(StringUtils.isAnyString(type) ? type : "root.pid"); + //item.setType(StringUtils.isAnyString(type) ? type : "root.pid"); if (json.has(TIMESTAMP_KEYWORD)) { String timestamp = json.getString(TIMESTAMP_KEYWORD); @@ -178,6 +213,11 @@ public static ReharvestItem fromJSON(JSONObject json) throws ParseException { item.setPodname(json.getString(POD_NAME_KEYWORD)); } + if (json.has(TYPE_KEYWORD)) { + String tt = json.getString(TYPE_KEYWORD); + item.setTypeOfReharvest(TypeOfReharvset.root.name().equals(tt) ? TypeOfReharvset.root : TypeOfReharvset.children); + } + return item; } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index 5f5d43cb00..36540f49b2 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -1,11 +1,15 @@ package cz.incad.kramerius.rest.apiNew.admin.v10.reharvest; +import java.io.IOException; import java.text.ParseException; + + import java.time.Instant; import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; +import javax.inject.Named; import javax.ws.rs.Consumes; import javax.ws.rs.DELETE; import javax.ws.rs.GET; @@ -20,10 +24,16 @@ import org.json.JSONArray; import org.json.JSONException; import org.json.JSONObject; +import org.w3c.dom.Document; +import org.w3c.dom.Element; import com.google.inject.Inject; +import cz.incad.kramerius.SolrAccess; import cz.incad.kramerius.rest.apiNew.exceptions.BadRequestException; +import cz.incad.kramerius.utils.XMLUtils; + +import static cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem.*; @Path("/admin/v7.0/reharvest") public class ReharvestResource { @@ -33,6 +43,11 @@ public class ReharvestResource { @Inject private ReharvestManager reharvestManager; + @Inject + @Named("new-index") + private SolrAccess solrAccess; + + @GET @Produces(MediaType.APPLICATION_JSON) public Response getHarvests() { @@ -73,10 +88,45 @@ public Response delete(@PathParam("id") String id) { @Consumes(MediaType.APPLICATION_JSON) public Response register(String json) { try { + JSONObject jsonObj = new JSONObject(json); - if (!jsonObj.has("id")) { - jsonObj.put("id", UUID.randomUUID().toString()); + if (!jsonObj.has(ID_KEYWORD)) { + jsonObj.put(ID_KEYWORD, UUID.randomUUID().toString()); + } + if (!jsonObj.has(PID_KEYWORD)) { + throw new BadRequestException(" Request must contain pid "); + } + if (!jsonObj.has(OWN_PID_PATH) || !jsonObj.has(ROOT_PID)) { + + Document solrDataByPid = this.solrAccess.getSolrDataByPid(jsonObj.getString(PID_KEYWORD)); + Element rootPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("str")) { + String fieldName = element.getAttribute("name"); + return fieldName.equals("root.pid"); + } + return false; + } + }); + Element ownPidPath = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("str")) { + String fieldName = element.getAttribute("name"); + return fieldName.equals("own_pid_path"); + } + return false; + } + }); + + jsonObj.put(ROOT_PID, rootPid.getTextContent()); + jsonObj.put(OWN_PID_PATH, ownPidPath.getTextContent()); + + } + + ReharvestItem item = ReharvestItem.fromJSON(jsonObj); if (item != null) { item.setTimestamp(Instant.now()); @@ -90,6 +140,9 @@ public Response register(String json) { return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); } catch(AlreadyRegistedPidsException e) { throw new BadRequestException(" Request contains already registered pids "+e.getPids()); + } catch (IOException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); } } @@ -108,7 +161,6 @@ public Response getTopHarvest(@QueryParam("state")String state) { } - @PUT @Path("{id}/state") @Produces(MediaType.APPLICATION_JSON) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java index 141aa39633..3cd5d1b235 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java @@ -1,6 +1,7 @@ package cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.impl; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -21,8 +22,8 @@ public MemoryReharvestManagerImpl() { } public void register(ReharvestItem item) throws AlreadyRegistedPidsException { - List alreadyRegistredPids = this.items.stream().filter(x -> x.getState() != null && (x.getState().equals("open") || x.getState().startsWith("waiting_for"))).map(ReharvestItem::getPids).flatMap(List::stream).collect(Collectors.toList()); - List intersection = new ArrayList<>(item.getPids()); + List alreadyRegistredPids = this.items.stream().filter(x -> x.getState() != null && (x.getState().equals("open") || x.getState().startsWith("waiting_for"))).map(ReharvestItem::getPid).collect(Collectors.toList()); + List intersection = Arrays.asList(item.getPid()); intersection.retainAll(alreadyRegistredPids); if (intersection.isEmpty()) { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java new file mode 100644 index 0000000000..7ae31d8923 --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java @@ -0,0 +1,178 @@ +package cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.impl; + +import java.io.StringWriter; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; +import java.text.ParseException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.ws.rs.core.MediaType; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.TransformerException; + +import org.apache.solr.client.solrj.SolrClient; +import org.json.JSONArray; +import org.json.JSONException; +import org.json.JSONObject; +import org.w3c.dom.DOMException; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientHandlerException; +import com.sun.jersey.api.client.UniformInterfaceException; +import com.sun.jersey.api.client.WebResource; + +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.AlreadyRegistedPidsException; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; +import cz.incad.kramerius.utils.BasicAuthenticationClientFilter; +import cz.incad.kramerius.utils.IOUtils; +import cz.incad.kramerius.utils.XMLUtils; +import cz.incad.kramerius.utils.conf.KConfiguration; + +public class SolrReharvestManagerImpl implements ReharvestManager { + + public static Logger LOGGER = Logger.getLogger(SolrReharvestManagerImpl.class.getName()); + + private Client client; + + public SolrReharvestManagerImpl() { + this.client = Client.create(); + } + + @Override + public void register(ReharvestItem item) throws AlreadyRegistedPidsException { + try { + String reharvest = KConfiguration.getInstance().getSolrReharvestHost(); + List pids = findAllRegistredPids(reharvest); + if (!pids.contains(item.getPid())) { + WebResource updateResource = this.client.resource(reharvest + "/update/json/docs?split=/&commit=true"); + String updated = updateResource.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON) + .entity(item.toJSON().toString(), MediaType.APPLICATION_JSON).post(String.class); + } else { + throw new AlreadyRegistedPidsException(Arrays.asList(item.getPid())); + } + } catch (UnsupportedEncodingException | UniformInterfaceException | ClientHandlerException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } + } + + private List findAllRegistredPids(String reharvest) throws UnsupportedEncodingException { + List pids = new ArrayList<>(); + String query = URLEncoder.encode("state:(open OR waiting_for)", "UTF-8"); + String fullUrl = String.format("%s/select?q=%s&rows=10000&fl=pid", reharvest, query); + String t = solrGet(fullUrl); + JSONArray docs = solrDocs(new JSONObject(t)); + for (int i = 0; i < docs.length(); i++) { + JSONObject doc = docs.getJSONObject(i); + if (doc.has("pid")) { + pids.add(doc.getString("pid")); + } + } + return pids; + } + + private String solrGet(String fullUrl) { + WebResource resource = this.client.resource(fullUrl); + String t = resource.accept(MediaType.APPLICATION_JSON).get(String.class); + return t; + } + + @Override + public List getItems() { + List items = new ArrayList<>(); + try { + String reharvest = KConfiguration.getInstance().getSolrReharvestHost(); + String sort = URLEncoder.encode("indexed asc", "UTF-8"); + String fullUrl = String.format("%s/select?q=*&rows=10000&sort=%s", reharvest,sort); + String t = solrGet(fullUrl); + JSONArray docs = solrDocs(new JSONObject(t)); + for (int i = 0; i < docs.length(); i++) { + JSONObject doc = docs.getJSONObject(i); + ReharvestItem reharvestItem = ReharvestItem.fromJSON(doc); + if (reharvestItem != null) + items.add(reharvestItem); + } + } catch (UniformInterfaceException | ClientHandlerException | JSONException | ParseException | UnsupportedEncodingException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } + return items; + } + + @Override + public ReharvestItem getTopItem(String status) { + try { + String reharvest = KConfiguration.getInstance().getSolrReharvestHost(); + String query = String.format("state:(%s)", status); + String sort = URLEncoder.encode("indexed asc", "UTF-8"); + + String fullUrl = String.format("%s/select?q=%s&rows=1&sort=%s", reharvest, query, sort); + String t = solrGet(fullUrl); + JSONArray docs = solrDocs(new JSONObject(t)); + if (docs.length() > 0) { + JSONObject doc = docs.getJSONObject(0); + ReharvestItem reharvestItem = ReharvestItem.fromJSON(doc); + return reharvestItem; + } + } catch (UniformInterfaceException | ClientHandlerException | JSONException | ParseException | UnsupportedEncodingException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } + return null; + } + + @Override + public ReharvestItem getItemById(String id) { + try { + String reharvest = KConfiguration.getInstance().getSolrReharvestHost(); + String query = String.format("id:(%s)", id); + String sort = URLEncoder.encode("indexed asc","UTF-8"); + String fullUrl = String.format("%s/select?q=%s&rows=1&sort=%s", reharvest, query,sort); + String t = solrGet(fullUrl); + JSONObject solrResp = new JSONObject(t); + JSONArray docs = solrDocs(solrResp); + if (docs.length() > 0) { + JSONObject doc = docs.getJSONObject(0); + ReharvestItem reharvestItem = ReharvestItem.fromJSON(doc); + return reharvestItem; + } + } catch (UniformInterfaceException | ClientHandlerException | JSONException | ParseException | UnsupportedEncodingException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } + return null; + } + + private JSONArray solrDocs(JSONObject solrResp) { + JSONArray docs = new JSONArray(); + if (solrResp.has("response")) { + docs = solrResp.getJSONObject("response").getJSONArray("docs"); + } + return docs; + } + + @Override + public void deregister(String id) { + try { + Document deleteDoc = XMLUtils.crateDocument("delete"); + Element idElm = deleteDoc.createElement("id"); + idElm.setTextContent(id); + deleteDoc.getDocumentElement().appendChild(idElm); + StringWriter writer = new StringWriter(); + XMLUtils.print(deleteDoc, writer); + + String reharvest = KConfiguration.getInstance().getSolrReharvestHost(); + + WebResource updateResource = this.client.resource(reharvest + "/update?commit=true"); + String updated = updateResource.accept(MediaType.APPLICATION_XML).type(MediaType.APPLICATION_XML) + .entity(writer.toString(), MediaType.APPLICATION_XML).post(String.class); + } catch (DOMException | UniformInterfaceException | ClientHandlerException | ParserConfigurationException + | TransformerException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } + } +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index b1e8178878..4f8723e07e 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -36,6 +36,7 @@ import cz.incad.kramerius.SolrAccess; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.AlreadyRegistedPidsException; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem.TypeOfReharvset; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.security.User; @@ -171,9 +172,35 @@ public boolean acceptElement(Element element) { return false; } }); - if (rootPid != null) { + Element ownPidPath = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("str")) { + String fieldName = element.getAttribute("name"); + return fieldName.equals("own_pid_path"); + } + return false; + } + }); + + Element ownPidPath = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("str")) { + String fieldName = element.getAttribute("name"); + return fieldName.equals("own_pid_path"); + } + return false; + } + }); + + + + if (rootPid != null && ownPidPath != null) { try { - ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", new ArrayList<>(Arrays.asList(rootPid.getTextContent().trim()))); + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", rootPid.getTextContent().trim(), ownPidPath.getTextContent().trim()); + // children + reharvestItem.setTypeOfReharvest(TypeOfReharvset.children); reharvestItem.setState("waiting_for_approve"); this.reharvestManager.register(reharvestItem); } catch (DOMException e) { diff --git a/shared/common/src/main/java/cz/incad/kramerius/solr/SolrModule.java b/shared/common/src/main/java/cz/incad/kramerius/solr/SolrModule.java index 3ef8ce182f..7c2bdddea1 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/solr/SolrModule.java +++ b/shared/common/src/main/java/cz/incad/kramerius/solr/SolrModule.java @@ -41,4 +41,12 @@ public SolrClient proxyUpdateClient() { String processingSolrHost = KConfiguration.getInstance().getSolrUpdatesHost(); return new ConcurrentUpdateSolrClient.Builder(processingSolrHost).withQueueSize(100).build(); } + +// @Provides +// @Named("proxyReharvest") +// @Singleton +// public SolrClient proxyReharvestClient() { +// String processingSolrHost = KConfiguration.getInstance().getSolrReharvestHost(); +// return new ConcurrentUpdateSolrClient.Builder(processingSolrHost).withQueueSize(100).build(); +// } } diff --git a/shared/common/src/main/java/cz/incad/kramerius/utils/conf/KConfiguration.java b/shared/common/src/main/java/cz/incad/kramerius/utils/conf/KConfiguration.java index b294afe73a..8094b7bd07 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/utils/conf/KConfiguration.java +++ b/shared/common/src/main/java/cz/incad/kramerius/utils/conf/KConfiguration.java @@ -134,6 +134,10 @@ public String getSolrUpdatesHost() { return getProperty("solrUpdatesHost"); } + public String getSolrReharvestHost() { + return getProperty("solrReharvestHost"); + } + public String getSolrSearchHost() { return getProperty("solrSearchHost"); } From e0bc9c8cff552240695a38218014e9a4bba4a89a Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 27 Jun 2024 08:24:22 +0200 Subject: [PATCH 151/235] Proxy handler support --- .../client/v60/redirection/ProxyHandlerSupport.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 4f8723e07e..9daec953dd 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -183,12 +183,12 @@ public boolean acceptElement(Element element) { } }); - Element ownPidPath = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { + Element ownParentPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { @Override public boolean acceptElement(Element element) { if (element.getNodeName().equals("str")) { String fieldName = element.getAttribute("name"); - return fieldName.equals("own_pid_path"); + return fieldName.equals("own_parent.pid"); } return false; } @@ -196,9 +196,9 @@ public boolean acceptElement(Element element) { - if (rootPid != null && ownPidPath != null) { + if (rootPid != null && ownPidPath != null && ownParentPid != null) { try { - ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", rootPid.getTextContent().trim(), ownPidPath.getTextContent().trim()); + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", ownParentPid.getTextContent().trim(), ownPidPath.getTextContent().trim()); // children reharvestItem.setTypeOfReharvest(TypeOfReharvset.children); reharvestItem.setState("waiting_for_approve"); From 6e3277e8a034a48c6c63c004aab60a97185a13e9 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 27 Jun 2024 09:54:28 +0200 Subject: [PATCH 152/235] Reharvets and update --- .../admin/v10/reharvest/ReharvestManager.java | 6 +++ .../v10/reharvest/ReharvestResource.java | 41 ++++++++++++------- .../impl/SolrReharvestManagerImpl.java | 30 ++++++++++++++ 3 files changed, 63 insertions(+), 14 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java index 50214d96f2..0835d07ba0 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java @@ -1,10 +1,16 @@ package cz.incad.kramerius.rest.apiNew.admin.v10.reharvest; +import java.io.UnsupportedEncodingException; +import java.text.ParseException; import java.util.List; +import org.json.JSONException; + public interface ReharvestManager { public void register(ReharvestItem item) throws AlreadyRegistedPidsException; + + public ReharvestItem update(ReharvestItem item) throws UnsupportedEncodingException, JSONException, ParseException; public List getItems(); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index 36540f49b2..b21141282a 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -1,6 +1,7 @@ package cz.incad.kramerius.rest.apiNew.admin.v10.reharvest; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.text.ParseException; @@ -165,26 +166,38 @@ public Response getTopHarvest(@QueryParam("state")String state) { @Path("{id}/state") @Produces(MediaType.APPLICATION_JSON) public Response changeState(@PathParam("id") String id, @QueryParam("state") String state) { - ReharvestItem itemById = reharvestManager.getItemById(id); - if (itemById != null) { - itemById.setState(state); - return Response.ok(itemById.toJSON().toString()).build(); - } else { - return Response.status(Response.Status.NOT_FOUND).build(); - } + try { + ReharvestItem itemById = reharvestManager.getItemById(id); + if (itemById != null) { + itemById.setState(state); + this.reharvestManager.update(itemById); + return Response.ok(itemById.toJSON().toString()).build(); + } else { + return Response.status(Response.Status.NOT_FOUND).build(); + } + } catch (UnsupportedEncodingException | JSONException | ParseException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); + } } @PUT @Path("{id}/pod") @Produces(MediaType.APPLICATION_JSON) public Response changePod(@PathParam("id") String id, @QueryParam("pod") String pod) { - ReharvestItem itemById = reharvestManager.getItemById(id); - if (itemById != null) { - itemById.setPodname(pod); - return Response.ok(itemById.toJSON().toString()).build(); - } else { - return Response.status(Response.Status.NOT_FOUND).build(); - } + try { + ReharvestItem itemById = reharvestManager.getItemById(id); + if (itemById != null) { + itemById.setPodname(pod); + this.reharvestManager.update(itemById); + return Response.ok(itemById.toJSON().toString()).build(); + } else { + return Response.status(Response.Status.NOT_FOUND).build(); + } + } catch (UnsupportedEncodingException | JSONException | ParseException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); + } } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java index 7ae31d8923..2e1b6fd8e4 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java @@ -63,6 +63,36 @@ public void register(ReharvestItem item) throws AlreadyRegistedPidsException { } } + + + + @Override + public ReharvestItem update(ReharvestItem item) { + try { + String reharvest = KConfiguration.getInstance().getSolrReharvestHost(); + WebResource updateResource = this.client.resource(reharvest + "/update/json/docs?split=/&commit=true"); + String updated = updateResource.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON) + .entity(item.toJSON().toString(), MediaType.APPLICATION_JSON).post(String.class); + return findItem(item.getId()); + } catch (UniformInterfaceException | ClientHandlerException | UnsupportedEncodingException | JSONException + | ParseException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } + return null; + } + + private ReharvestItem findItem(String id) throws UnsupportedEncodingException, JSONException, ParseException { + String reharvest = KConfiguration.getInstance().getSolrReharvestHost(); + List pids = new ArrayList<>(); + String query = URLEncoder.encode("state:(%s)", "UTF-8"); + String fullUrl = String.format("%s/select?q=id:%ss&rows=1", reharvest, id); + String t = solrGet(fullUrl); + JSONArray docs = solrDocs(new JSONObject(t)); + if (docs.length() > 0) { + return ReharvestItem.fromJSON(docs.getJSONObject(0)); + } else return null; + } + private List findAllRegistredPids(String reharvest) throws UnsupportedEncodingException { List pids = new ArrayList<>(); String query = URLEncoder.encode("state:(open OR waiting_for)", "UTF-8"); From c84579daeebade69ea50ebba031f9319c17bc025 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 27 Jun 2024 10:04:27 +0200 Subject: [PATCH 153/235] MemoryReharvestManager --- .../v10/reharvest/impl/MemoryReharvestManagerImpl.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java index 3cd5d1b235..1cc4820735 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java @@ -1,5 +1,7 @@ package cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.impl; +import java.io.UnsupportedEncodingException; +import java.text.ParseException; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; @@ -7,6 +9,8 @@ import java.util.Map; import java.util.stream.Collectors; +import org.json.JSONException; + import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.AlreadyRegistedPidsException; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; @@ -57,6 +61,12 @@ public ReharvestItem getItemById(String id) { return this.mapper.get(id); } + + @Override + public ReharvestItem update(ReharvestItem item) throws UnsupportedEncodingException, JSONException, ParseException { + return item; + } + public void deregister(String id) { ReharvestItem ritem = this.mapper.get(id); if (ritem != null) { From a33362ef55dd5a7d1f523439b0354e0ef0cb856c Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 27 Jun 2024 19:04:36 +0200 Subject: [PATCH 154/235] Changed ReharvestItem --- .../kramerius/KubernetesReharvestProcess.java | 2 +- .../incad/kramerius/utils/ReharvestUtils.java | 41 ++++++++++++++++--- .../v60/redirection/ProxyHandlerSupport.java | 13 +++++- 3 files changed, 48 insertions(+), 8 deletions(-) diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java index 5c3da9b842..6129712f82 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -108,7 +108,7 @@ public static void main(String[] args) throws ParserConfigurationException, Ille if (proxyURl != null) { Map configurations = libraryConfigurations(client, proxyURl); - ReharvestUtils.reharvestPIDFromGivenCollections(pid, configurations, ""+onlyShowConfiguration, destinationMap, iterationMap); + ReharvestUtils.reharvestPIDFromGivenCollections(pid, configurations, ""+onlyShowConfiguration, destinationMap, iterationMap, ReharvestItem.fromJSON(itemObject)); if (!onlyShowConfiguration) { changeState(client, wurl, id,"closed"); diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index ce8822c535..89a61a71d3 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -176,15 +176,37 @@ public static String renderTemplate(String api, Map iteration, Ma return configuration; } - public static String fq(String api, String pid) { - switch(api) { - case "v7": return "root.pid:\""+pid+"\""; - case "v5": return "root_pid:\""+pid+"\""; + public static String fq(String api, String pid, ReharvestItem item) throws UnsupportedEncodingException { + switch(item.getTypeOfReharvest()) { + case root: + switch(api) { + case "v7": return "root.pid:\""+pid+"\""; + case "v5": return "root_pid:\""+pid+"\""; + } + return "root_pid:\""+pid+"\""; + + case children: + + switch(api) { + case "v7": { + String ownPidPath = item.getOwnPidPath().replaceAll(":", "\\\\:")+"/*"; + String fq = String.format("own_pid_path:%s", ownPidPath); + return fq; + } + case "v5": { + String pidPath = item.getOwnPidPath().replaceAll(":", "\\\\:")+"/*"; + String fq = String.format("pid_path:%s", pidPath); + return fq; + } + } + return "root_pid:\""+pid+"\""; + + } return "root_pid:\""+pid+"\""; } - public static void reharvestPIDFromGivenCollections(String pid, Map collectionConfigurations, String onlyShowConfiguration, Map destinationMap, Map iterationMap) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchMethodException, MigrateSolrIndexException, IOException, ParserConfigurationException, SAXException { + public static void reharvestPIDFromGivenCollections(String pid, Map collectionConfigurations, String onlyShowConfiguration, Map destinationMap, Map iterationMap, ReharvestItem item) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchMethodException, MigrateSolrIndexException, IOException, ParserConfigurationException, SAXException { List harvestFiles = new ArrayList<>(); for (String ac : collectionConfigurations.keySet()) { try { @@ -211,7 +233,7 @@ public static void reharvestPIDFromGivenCollections(String pid, Map= 0) { + pidPath = pidPath.substring(0, index + ownParentPidText.length()).trim(); + } + + try { - ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", ownParentPid.getTextContent().trim(), ownPidPath.getTextContent().trim()); + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", ownParentPid.getTextContent().trim(), pidPath.getTextContent().trim()); // children reharvestItem.setTypeOfReharvest(TypeOfReharvset.children); reharvestItem.setState("waiting_for_approve"); From e7a9d5492b21aef1edbd3ce4b3e1caf5ed6281c0 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 27 Jun 2024 19:23:17 +0200 Subject: [PATCH 155/235] Changed condition --- .../apiNew/client/v60/redirection/ProxyHandlerSupport.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 987c7146d5..a41768b776 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -199,17 +199,13 @@ public boolean acceptElement(Element element) { if (rootPid != null && ownPidPath != null && ownParentPid != null) { String pidPath = ownPidPath.getTextContent().trim(); - //String pidPath = ownPidPath.getTextContent().trim(); String ownParentPidText = ownParentPid.getTextContent().trim(); - int index = pidPath.indexOf(ownParentPidText); if (index >= 0) { pidPath = pidPath.substring(0, index + ownParentPidText.length()).trim(); } - - try { - ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", ownParentPid.getTextContent().trim(), pidPath.getTextContent().trim()); + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", ownParentPid.getTextContent().trim(), pidPath); // children reharvestItem.setTypeOfReharvest(TypeOfReharvset.children); reharvestItem.setState("waiting_for_approve"); From a36ab4b76a0bee324b25b9910793bd4a687cd563 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 28 Jun 2024 11:41:49 +0200 Subject: [PATCH 156/235] Changed in process & OAI --- .../main/java/cz/incad/kramerius/utils/ReharvestUtils.java | 7 ++++--- .../src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index 89a61a71d3..1c847f7364 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -98,7 +98,8 @@ public static List> findPidByType(Map iterat break; case children: String ownPidPath = item.getOwnPidPath(); - ownPidPath = ownPidPath.replaceAll(":", "\\\\:")+"/*"; + //ownPidPath = ownPidPath.replaceAll(":", "\\\\:")+"/*"; + ownPidPath = ownPidPath.replaceAll(":", "\\\\:")+"*"; filterQuery = String.format("own_pid_path:%s", ownPidPath);; break; } @@ -189,12 +190,12 @@ public static String fq(String api, String pid, ReharvestItem item) throws Unsup switch(api) { case "v7": { - String ownPidPath = item.getOwnPidPath().replaceAll(":", "\\\\:")+"/*"; + String ownPidPath = item.getOwnPidPath().replaceAll(":", "\\\\:")+"*"; String fq = String.format("own_pid_path:%s", ownPidPath); return fq; } case "v5": { - String pidPath = item.getOwnPidPath().replaceAll(":", "\\\\:")+"/*"; + String pidPath = item.getOwnPidPath().replaceAll(":", "\\\\:")+"*"; String fq = String.format("pid_path:%s", pidPath); return fq; } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java index 9f57a1c05c..7eb32b038a 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java @@ -229,12 +229,14 @@ public void perform(Provider userProvider,Provider clientProvider, if (set != null) { selectedSet = sets.findBySet(set); if (selectedSet ==null) { + LOGGER.severe(String.format("Cannot find set %s",set)); throw new OAIException(ErrorCode.badArgument, OAIVerb.ListRecords, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); } } else if (resumptionToken != null){ selectedSet = sets.findByToken(resumptionToken); metadataPrefix = OAITools.metadataFromResumptionToken(resumptionToken); if (metadataPrefix == null || MetadataExport.findByPrefix(metadataPrefix) == null) { + LOGGER.severe(String.format("Bad resumption token %s",resumptionToken)); throw new OAIException(ErrorCode.badResumptionToken, OAIVerb.ListRecords, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); } } @@ -246,7 +248,7 @@ public void perform(Provider userProvider,Provider clientProvider, selectedSet = sets.getDefaultSet(); } - Element requestElement = OAITools.requestElement(doc, OAIVerb.ListIdentifiers, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); + Element requestElement = OAITools.requestElement(doc, OAIVerb.ListRecords, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); doc.getDocumentElement().appendChild(requestElement); Element identify = doc.createElement("ListRecords"); From 6c93e994d988679d6ed5dcb3114cb746f50b4b67 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 28 Jun 2024 11:45:12 +0200 Subject: [PATCH 157/235] Sets - pirnt configurations --- rest/src/main/java/cz/incad/kramerius/rest/oai/OAISets.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISets.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISets.java index ce508419a1..00fd2a09b5 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISets.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISets.java @@ -24,6 +24,7 @@ import java.util.Map; import java.util.Optional; import java.util.logging.Logger; +import java.util.stream.Collectors; import cz.incad.kramerius.rest.apiNew.ConfigManager; import cz.incad.kramerius.utils.conf.KConfiguration; @@ -201,6 +202,8 @@ private void loadOAISetPropertiesFromConf(String key, String[] values, OAISet ps } public OAISet findBySet(String setName) { + LOGGER.info(String.format("Finding %s",setName)); + LOGGER.info("ALL sets " + this.sets.stream().map(OAISet::getSetSpec).collect(Collectors.toList())); if (setName != null && !setName.equals(DEFAULT_SET_KEY)) { Optional found = this.sets.stream() .filter(oaiSet -> oaiSet.getSetSpec().equals(setName)) From ad48f6374923cbdfc261391eacda50c8d7de66f5 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 28 Jun 2024 12:45:41 +0200 Subject: [PATCH 158/235] OAI - delete --- .../main/java/cz/incad/kramerius/rest/oai/OAIVerb.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java index 7eb32b038a..2139ebb715 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java @@ -286,9 +286,13 @@ public void perform(Provider userProvider,Provider clientProvider, Element record= doc.createElement("record"); Element header = oaiRec.toHeader(doc, selectedSet.getSetSpec()); - Element metadata = doc.createElement("metadata"); - metadata.appendChild(oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,selectedSet)); + Element metadataElm = oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,selectedSet); + if (metadataElm != null) { + metadata.appendChild(metadataElm); + } else { + record.setAttribute("deleted","true"); + } record.appendChild(header); record.appendChild(metadata); From 4e3abe805d73d908e35c743bc789fe974a9539ee Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 2 Jul 2024 10:02:31 +0200 Subject: [PATCH 159/235] Reharvets --- .../cz/incad/kramerius/KubernetesReharvestProcess.java | 2 +- .../java/cz/incad/kramerius/utils/ReharvestUtils.java | 9 +++++++-- .../apiNew/admin/v10/reharvest/ReharvestResource.java | 9 ++++++--- 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java index 6129712f82..665371d144 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -53,7 +53,7 @@ protected static Client buildClient() { return Client.create(cc); } - public static void main(String[] args) throws ParserConfigurationException, IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchMethodException, MigrateSolrIndexException, IOException, SAXException { + public static void main(String[] args) { TimeZone.setDefault(TimeZone.getTimeZone("Europe/Prague")); diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index 1c847f7364..1b811411df 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -45,6 +45,7 @@ public class ReharvestUtils { + private static final String ITERATION_ROWS_STRING_VALUE = "300"; public static final Logger LOGGER = Logger.getLogger(ReharvestUtils.class.getName()); private ReharvestUtils() {} @@ -110,7 +111,9 @@ public static List> findPidByType(Map iterat String queryCursorMark = null; do { - Element element = pidsCursorQuery(client, iterationUrl, masterQuery, cursorMark, 3000, + String sRows = iterationMap.containsKey("rows") ? iterationMap.get("rows") : ITERATION_ROWS_STRING_VALUE ; + LOGGER.info( String.format("Number of iteration rows %s", sRows)); + Element element = pidsCursorQuery(client, iterationUrl, masterQuery, cursorMark, Integer.parseInt(sRows), filterQuery, "select", "compositeId+pid", "compositeId asc", "", ""); cursorMark = findCursorMark(element); queryCursorMark = findQueryCursorMark(element); @@ -244,6 +247,7 @@ public static void reharvestPIDFromGivenCollections(String pid, Map Date: Tue, 2 Jul 2024 10:28:53 +0200 Subject: [PATCH 160/235] Reharvests - no root --- .../rest/apiNew/admin/v10/reharvest/ReharvestResource.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index 314634fdb0..d3c6ae1526 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -124,6 +124,8 @@ public boolean acceptElement(Element element) { if (rootPid != null) { jsonObj.put(ROOT_PID, rootPid.getTextContent()); + } else { + jsonObj.put(ROOT_PID, jsonObj.getString(PID_KEYWORD)); } if (ownPidPath != null) { jsonObj.put(OWN_PID_PATH, ownPidPath.getTextContent()); From e2248b3636329ffe92f834e92aa9809938f6bcb2 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 2 Jul 2024 13:46:05 +0200 Subject: [PATCH 161/235] Reharvests --- .../src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index 1b811411df..fe9ad93b61 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -99,6 +99,9 @@ public static List> findPidByType(Map iterat break; case children: String ownPidPath = item.getOwnPidPath(); + if (!StringUtils.isAnyString(ownPidPath)) { + throw new IllegalArgumentException("own pid path is null string"); + } //ownPidPath = ownPidPath.replaceAll(":", "\\\\:")+"/*"; ownPidPath = ownPidPath.replaceAll(":", "\\\\:")+"*"; filterQuery = String.format("own_pid_path:%s", ownPidPath);; From 363345a11ed52bb42dccf4025b996814ebbae7af Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 2 Jul 2024 13:47:51 +0200 Subject: [PATCH 162/235] Reharvests chnages --- .../admin/v10/reharvest/ReharvestResource.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index d3c6ae1526..b9a237759e 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -99,7 +99,8 @@ public Response register(String json) { } if (!jsonObj.has(OWN_PID_PATH) || !jsonObj.has(ROOT_PID)) { - + + Document solrDataByPid = this.solrAccess.getSolrDataByPid(jsonObj.getString(PID_KEYWORD)); Element rootPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { @Override @@ -135,9 +136,13 @@ public boolean acceptElement(Element element) { ReharvestItem item = ReharvestItem.fromJSON(jsonObj); if (item != null) { - item.setTimestamp(Instant.now()); - this.reharvestManager.register(item); - return Response.ok(item.toJSON().toString()).build(); + if (item.getTypeOfReharvest().equals(ReharvestItem.TypeOfReharvset.children) && !jsonObj.has(OWN_PID_PATH)) { + return Response.status(Response.Status.BAD_REQUEST).build(); + } else { + item.setTimestamp(Instant.now()); + this.reharvestManager.register(item); + return Response.ok(item.toJSON().toString()).build(); + } } else { return Response.status(Response.Status.NOT_FOUND).build(); } From 9c5fdf441692a934c2a71259f1361c88879c927b Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 2 Jul 2024 14:49:49 +0200 Subject: [PATCH 163/235] Reharvest items; new type of reharvests --- .../admin/v10/reharvest/ReharvestItem.java | 11 +++++----- .../v10/reharvest/ReharvestResource.java | 21 +++++++------------ 2 files changed, 14 insertions(+), 18 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java index c9b6fa0b85..2a0a69a432 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java @@ -21,13 +21,14 @@ public class ReharvestItem { public static enum TypeOfReharvset { - root, children; + + root, // kompletni reharvest celeho titulu + children, // reharvest titulu a potomku + + fix_children, // dilo v cdk smazano a je potreba stahnout z krameriu informace o detech a titulu a pak pomoci own_pid_path + fix_root; // dilo je v cdk smazano a je potreba stahnot z krameriu informace o korenovem dile } -// static enum ReharvestItemState { -// open, closed; -// } -// public static FastDateFormat FORMAT = FastDateFormat.getInstance("yyyy-MM-dd'T'HH:mm:ss.sss'Z'", TimeZone.getTimeZone("UTC")); public static final String TIMESTAMP_KEYWORD = "indexed"; diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index b9a237759e..e23104a102 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -32,6 +32,7 @@ import cz.incad.kramerius.SolrAccess; import cz.incad.kramerius.rest.apiNew.exceptions.BadRequestException; +import cz.incad.kramerius.utils.StringUtils; import cz.incad.kramerius.utils.XMLUtils; import static cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem.*; @@ -89,7 +90,7 @@ public Response delete(@PathParam("id") String id) { @Consumes(MediaType.APPLICATION_JSON) public Response register(String json) { try { - + // switch reharvest vs new harvest JSONObject jsonObj = new JSONObject(json); if (!jsonObj.has(ID_KEYWORD)) { jsonObj.put(ID_KEYWORD, UUID.randomUUID().toString()); @@ -125,26 +126,20 @@ public boolean acceptElement(Element element) { if (rootPid != null) { jsonObj.put(ROOT_PID, rootPid.getTextContent()); - } else { - jsonObj.put(ROOT_PID, jsonObj.getString(PID_KEYWORD)); } if (ownPidPath != null) { jsonObj.put(OWN_PID_PATH, ownPidPath.getTextContent()); } } - + // nasel to v indexu (pro polozky, ktere nejsou v indexu je potreba novy typ harvestu ReharvestItem item = ReharvestItem.fromJSON(jsonObj); - if (item != null) { - if (item.getTypeOfReharvest().equals(ReharvestItem.TypeOfReharvset.children) && !jsonObj.has(OWN_PID_PATH)) { - return Response.status(Response.Status.BAD_REQUEST).build(); - } else { - item.setTimestamp(Instant.now()); - this.reharvestManager.register(item); - return Response.ok(item.toJSON().toString()).build(); - } + if (item != null && StringUtils.isAnyString(item.getRootPid()) && StringUtils.isAnyString(item.getOwnPidPath())) { + item.setTimestamp(Instant.now()); + this.reharvestManager.register(item); + return Response.ok(item.toJSON().toString()).build(); } else { - return Response.status(Response.Status.NOT_FOUND).build(); + return Response.status(Response.Status.BAD_REQUEST).build(); } } catch (JSONException | ParseException e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); From 3507a51863609f2d18f2f46dfed22666a34871bb Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 3 Jul 2024 13:45:35 +0200 Subject: [PATCH 164/235] Reharvests changes --- .../kramerius/KubernetesReharvestProcess.java | 19 ++++++++---- .../admin/v10/reharvest/ReharvestItem.java | 26 ++++++++++++---- .../v10/reharvest/ReharvestResource.java | 31 ++++++++++++++++--- 3 files changed, 60 insertions(+), 16 deletions(-) diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java index 665371d144..ad477f2f2c 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -106,9 +106,10 @@ public static void main(String[] args) { // https://api.val.ceskadigitalniknihovna.cz/search/api/admin/v7.0/connected/ String proxyURl = proxyMap.get("url"); if (proxyURl != null) { - Map configurations = libraryConfigurations(client, proxyURl); + ReharvestItem reharvestItem = ReharvestItem.fromJSON(itemObject); - ReharvestUtils.reharvestPIDFromGivenCollections(pid, configurations, ""+onlyShowConfiguration, destinationMap, iterationMap, ReharvestItem.fromJSON(itemObject)); + Map configurations = libraryConfigurations(client, proxyURl,reharvestItem); + ReharvestUtils.reharvestPIDFromGivenCollections(pid, configurations, ""+onlyShowConfiguration, destinationMap, iterationMap, reharvestItem); if (!onlyShowConfiguration) { changeState(client, wurl, id,"closed"); @@ -160,7 +161,7 @@ private static void changePodname(Client client, String wurl, String id, String } } - public static Map libraryConfigurations(Client client, String proxyURl) { + public static Map libraryConfigurations(Client client, String proxyURl, ReharvestItem reharvestItem) { Map configurations = new HashMap<>(); WebResource proxyWebResource = client.resource(proxyURl); ClientResponse allConnectedItems = proxyWebResource.accept(MediaType.APPLICATION_JSON) @@ -171,6 +172,7 @@ public static Map libraryConfigurations(Client client, Strin for (Object key : responseAllConnectedObject.keySet()) { JSONObject lib = responseAllConnectedObject.getJSONObject(key.toString()); if (lib.has("status") && lib.getBoolean("status")) { + String configURl = proxyURl; if (!configURl.endsWith("/")) { @@ -182,9 +184,14 @@ public static Map libraryConfigurations(Client client, Strin ClientResponse configReourceStatus = configResource.accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); if (configReourceStatus.getStatus() == ClientResponse.Status.OK.getStatusCode()) { - - configurations.put(key.toString(), - new JSONObject(configReourceStatus.getEntity(String.class))); + boolean add = true; + if (reharvestItem.getLibraries() != null & !reharvestItem.getLibraries().isEmpty()) { + add = reharvestItem.getLibraries().contains(key.toString()); + } + if (add) { + configurations.put(key.toString(), + new JSONObject(configReourceStatus.getEntity(String.class))); + } } } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java index 2a0a69a432..3baeda4b3d 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java @@ -25,11 +25,10 @@ public static enum TypeOfReharvset { root, // kompletni reharvest celeho titulu children, // reharvest titulu a potomku - fix_children, // dilo v cdk smazano a je potreba stahnout z krameriu informace o detech a titulu a pak pomoci own_pid_path - fix_root; // dilo je v cdk smazano a je potreba stahnot z krameriu informace o korenovem dile + new_children, // dilo v cdk smazano a je potreba stahnout z krameriu informace o detech a titulu a pak pomoci own_pid_path + new_root; // dilo je v cdk smazano a je potreba stahnot z krameriu informace o korenovem dile } - - + public static final String TIMESTAMP_KEYWORD = "indexed"; public static final String PID_KEYWORD = "pid"; @@ -42,6 +41,9 @@ public static enum TypeOfReharvset { public static final String ID_KEYWORD = "id"; public static final String POD_NAME_KEYWORD = "pod"; + public static final String LIBRARIES_KEYWORD = "libraries"; + public static final String ERROR_MESSAGE_KEYWORD="error"; + // unique identifier private String id; private String name; @@ -108,8 +110,8 @@ public List getLibraries() { public void setLibraries(List libraries) { this.libraries = libraries; + } - public Instant getTimestamp() { return timestamp; } @@ -180,6 +182,12 @@ public JSONObject toJSON() { obj.put(ROOT_PID, this.rootPid); } + if (this.libraries != null && !this.libraries.isEmpty()) { + JSONArray jsonArray = new JSONArray(); + this.libraries.forEach(jsonArray::put); + obj.put(LIBRARIES_KEYWORD, jsonArray); + } + obj.put(TYPE_KEYWORD, this.typeOfReharvest.name()); obj.put(TIMESTAMP_KEYWORD,DateTimeFormatter.ISO_INSTANT.format(this.timestamp)); @@ -194,7 +202,6 @@ public static ReharvestItem fromJSON(JSONObject json) throws ParseException { String rootPid = json.optString(ROOT_PID); String state = json.optString(STATE_KEYWORD); - //String type = json.optString(TYPE_KEYWORD); ReharvestItem item = new ReharvestItem(id); item.setName(name); item.setPid(pid); @@ -218,6 +225,13 @@ public static ReharvestItem fromJSON(JSONObject json) throws ParseException { String tt = json.getString(TYPE_KEYWORD); item.setTypeOfReharvest(TypeOfReharvset.root.name().equals(tt) ? TypeOfReharvset.root : TypeOfReharvset.children); } + + if (json.has(LIBRARIES_KEYWORD)) { + JSONArray libsArray = json.getJSONArray(LIBRARIES_KEYWORD); + List libs = new ArrayList<>(); + for (int i = 0; i < libsArray.length(); i++) { libs.add(libsArray.getString(i));} + item.setLibraries(libs); + } return item; } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index e23104a102..51c008a38e 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -6,9 +6,11 @@ import java.time.Instant; +import java.util.List; import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; +import java.util.stream.Collectors; import javax.inject.Named; import javax.ws.rs.Consumes; @@ -84,6 +86,8 @@ public Response delete(@PathParam("id") String id) { return Response.status(Response.Status.NOT_FOUND).build(); } } + + @PUT @Produces(MediaType.APPLICATION_JSON) @@ -97,11 +101,8 @@ public Response register(String json) { } if (!jsonObj.has(PID_KEYWORD)) { throw new BadRequestException(" Request must contain pid "); - } if (!jsonObj.has(OWN_PID_PATH) || !jsonObj.has(ROOT_PID)) { - - Document solrDataByPid = this.solrAccess.getSolrDataByPid(jsonObj.getString(PID_KEYWORD)); Element rootPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { @Override @@ -123,6 +124,18 @@ public boolean acceptElement(Element element) { return false; } }); + + Element cdkCollection = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("arr")) { + String fieldName = element.getAttribute("name"); + return fieldName.equals("collection"); + } + return false; + } + }); + if (rootPid != null) { jsonObj.put(ROOT_PID, rootPid.getTextContent()); @@ -130,16 +143,26 @@ public boolean acceptElement(Element element) { if (ownPidPath != null) { jsonObj.put(OWN_PID_PATH, ownPidPath.getTextContent()); } + + if (cdkCollection != null) { + List collections = XMLUtils.getElements(cdkCollection).stream().map(Element::getTextContent).collect(Collectors.toList()); + JSONArray jsonArr = new JSONArray(); + collections.forEach(jsonArr::put); + jsonObj.put(LIBRARIES_KEYWORD, jsonArr); + } } // nasel to v indexu (pro polozky, ktere nejsou v indexu je potreba novy typ harvestu ReharvestItem item = ReharvestItem.fromJSON(jsonObj); + // check if (item != null && StringUtils.isAnyString(item.getRootPid()) && StringUtils.isAnyString(item.getOwnPidPath())) { item.setTimestamp(Instant.now()); this.reharvestManager.register(item); return Response.ok(item.toJSON().toString()).build(); } else { - return Response.status(Response.Status.BAD_REQUEST).build(); + JSONObject errorObject = new JSONObject(); + errorObject.put("error", "No root pid or own_pid_path"); + return Response.status(Response.Status.BAD_REQUEST).entity(errorObject.toString()).type(MediaType.APPLICATION_JSON_TYPE).build(); } } catch (JSONException | ParseException e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); From 8919c3017071ceb11aec9c17f2ece02dfdf2af15 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 3 Jul 2024 14:42:09 +0200 Subject: [PATCH 165/235] Reharvest process and resource --- .../kramerius/KubernetesReharvestProcess.java | 9 ++-- .../incad/kramerius/utils/ReharvestUtils.java | 52 ++++++++++++++----- .../v10/reharvest/ReharvestResource.java | 2 +- 3 files changed, 46 insertions(+), 17 deletions(-) diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java index ad477f2f2c..7166870b20 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -99,16 +99,17 @@ public static void main(String[] args) { List> allPidsList = ReharvestUtils.findPidByType(iterationMap, client, ReharvestItem.fromJSON(itemObject)); // check size; if size > 10000 - fail state if (allPidsList.size() < maxItemsToDelete) { - // delete all pids - ReharvestUtils.deleteAllGivenPids(client, destinationMap, allPidsList, onlyShowConfiguration); - // List acronyms = new ArrayList(); - // https://api.val.ceskadigitalniknihovna.cz/search/api/admin/v7.0/connected/ String proxyURl = proxyMap.get("url"); if (proxyURl != null) { ReharvestItem reharvestItem = ReharvestItem.fromJSON(itemObject); Map configurations = libraryConfigurations(client, proxyURl,reharvestItem); + // check channels before delete + ReharvestUtils.checkChannelsBeforeDelete(client, configurations); + // delete all asociated pids from index + ReharvestUtils.deleteAllGivenPids(client, destinationMap, allPidsList, onlyShowConfiguration); + // reindex pids ReharvestUtils.reharvestPIDFromGivenCollections(pid, configurations, ""+onlyShowConfiguration, destinationMap, iterationMap, reharvestItem); if (!onlyShowConfiguration) { diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index fe9ad93b61..a68e1239c4 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -21,6 +21,7 @@ import java.util.logging.Logger; import java.util.stream.Collectors; +import javax.ws.rs.core.MediaType; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; @@ -36,6 +37,8 @@ import com.google.common.io.Files; import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.KubernetesReharvestProcess; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; @@ -213,31 +216,43 @@ public static String fq(String api, String pid, ReharvestItem item) throws Unsup return "root_pid:\""+pid+"\""; } + public static void checkChannelsBeforeDelete(Client client, Map collectionConfigurations) { + for (String ac : collectionConfigurations.keySet()) { + JSONObject colObject = collectionConfigurations.get(ac); + String apiVersion = colObject.optString("api","v5"); + if (!colObject.has("forwardurl")) { + LOGGER.severe(String.format("Skipping %s", ac)); + continue; + } + String channel = colObject.optString("forwardurl"); + String fullChannelUrl = solrChannelUrl(apiVersion, channel); + + WebResource configResource = client.resource(fullChannelUrl+"/select?q=*&rows=0&wt=json"); + ClientResponse configReourceStatus = configResource.accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + if (configReourceStatus.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + // ok - live channel + } else throw new IllegalStateException(String.format("Channel for %s(%s) doesnt work ", ac, channel)); + } + } + public static void reharvestPIDFromGivenCollections(String pid, Map collectionConfigurations, String onlyShowConfiguration, Map destinationMap, Map iterationMap, ReharvestItem item) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchMethodException, MigrateSolrIndexException, IOException, ParserConfigurationException, SAXException { List harvestFiles = new ArrayList<>(); for (String ac : collectionConfigurations.keySet()) { try { JSONObject colObject = collectionConfigurations.get(ac); + String apiVersion = colObject.optString("api","v5"); if (!colObject.has("forwardurl")) { LOGGER.severe(String.format("Skipping %s", ac)); continue; } + Map iteration = new HashMap<>(iterationMap); String channel = colObject.optString("forwardurl"); + String fullChannelUrl = solrChannelUrl(apiVersion, channel); - Map iteration = new HashMap<>(iterationMap); - //http://mzk-tunnel.cdk-proxy.svc.cluster.local/search" - //http://knav-tunnel.cdk-proxy.svc.cluster.local/search/api/v5.0/cdk/forward/sync/solr - //v7.0 - //search/api/cdk/v7.0/forward/sync/solr/ - if (apiVersion.toLowerCase().equals("v5")) { - //channel = - iteration.put("url", channel+(channel.endsWith("/") ? "" : "/")+"api/v5.0/cdk/forward/sync/solr"); - } else { - iteration.put("url", channel+(channel.endsWith("/") ? "" : "/")+"api/cdk/v7.0/forward/sync/solr"); - } - + iteration.put("url", fullChannelUrl); iteration.put("dl", ac); iteration.put("fquery", fq(apiVersion, pid, item)); @@ -275,6 +290,19 @@ public static void reharvestPIDFromGivenCollections(String pid, Map Date: Fri, 5 Jul 2024 08:20:56 +0200 Subject: [PATCH 166/235] Reharvests = changed endpoint --- .../services/iterators/IterationItem.java | 7 + .../iterators/solr/SolrPageIterator.java | 8 +- .../kramerius/services/utils/SolrUtils.java | 4 +- .../kramerius/KubernetesReharvestProcess.java | 57 ++++--- .../cz/incad/kramerius/TooBigException.java | 16 ++ .../incad/kramerius/utils/ReharvestUtils.java | 139 +++++++++--------- .../v10/reharvest/ReharvestResource.java | 2 +- .../rest/apiNew/client/v60/ItemsResource.java | 23 +++ .../v60/redirection/ProxyHandlerSupport.java | 2 +- 9 files changed, 155 insertions(+), 103 deletions(-) create mode 100644 processes/cdkreharvest/src/main/java/cz/incad/kramerius/TooBigException.java diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/IterationItem.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/IterationItem.java index 69d3965164..118b9f31fb 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/IterationItem.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/IterationItem.java @@ -44,4 +44,11 @@ public boolean equals(Object o) { public int hashCode() { return Objects.hash(getPid(), getSource()); } + + @Override + public String toString() { + return "IterationItem [pid=" + pid + ", source=" + source + "]"; + } + + } diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/solr/SolrPageIterator.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/solr/SolrPageIterator.java index 4469155149..a9898656b5 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/solr/SolrPageIterator.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/solr/SolrPageIterator.java @@ -31,12 +31,12 @@ public SolrPageIterator(String address, String masterQuery, String filterQuery, super(address, masterQuery, filterQuery, endpoint, id, sorting, rows, user, pass); } - public static Element paginationQuery(Client client, String url, String mq, String offset, int rows, String filterQuery, String endpoint, String user, String pass) throws IOException, SAXException, ParserConfigurationException { + public static Element paginationQuery(Client client, String url, String mq, String offset, int rows, String filterQuery, String endpoint, String identifierField, String user, String pass) throws IOException, SAXException, ParserConfigurationException { String fullQuery = null; if (StringUtils.isAnyString(filterQuery)) { - fullQuery = String.format("?q=%s&start=%s&rows=%d&fq=%s&fl=PID",mq,offset, rows, URLEncoder.encode(filterQuery,"UTF-8")); + fullQuery = String.format("?q=%s&start=%s&rows=%d&fq=%s&fl=%s",mq,offset, rows, URLEncoder.encode(filterQuery,"UTF-8"), identifierField); } else { - fullQuery = String.format("?q=%s&start=%s&rows=%d&fl=PID",mq,offset, rows); + fullQuery = String.format("?q=%s&start=%s&rows=%d&fl=%s",mq,offset, rows,identifierField); } String query = endpoint+ fullQuery+"&wt=xml"; return SolrUtils.executeQuery(client, url, query, user, pass); @@ -64,7 +64,7 @@ public void iterate(Client client, ProcessIterationCallback iterationCallback, P int offset = 0; int numberOfResult = Integer.MAX_VALUE; do { - Element element = paginationQuery( client, address,masterQuery, ""+offset, rows, filterQuery, endpoint, this.user, this.pass); + Element element = paginationQuery( client, address,masterQuery, ""+offset, rows, filterQuery, endpoint, id, this.user, this.pass); if (numberOfResult == Integer.MAX_VALUE) { numberOfResult = findNumberOfResults(element); } diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/SolrUtils.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/SolrUtils.java index 7141423228..4ddd135930 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/SolrUtils.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/SolrUtils.java @@ -156,7 +156,9 @@ public boolean acceptElement(Element element) { }); return elements.stream().map(item -> { - Element str = XMLUtils.findElement(item, new XMLUtils.ElementsFilter() { + + //LOGGER.info("Item") + Element str = XMLUtils.findElement(item, new XMLUtils.ElementsFilter() { @Override public boolean acceptElement(Element element) { return element.getNodeName().equals("str"); diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java index 7166870b20..2aec02e878 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -1,7 +1,9 @@ package cz.incad.kramerius; import java.io.IOException; +import java.io.StringWriter; import java.io.UnsupportedEncodingException; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -95,32 +97,39 @@ public static void main(String[] args) { } } - // find all pids - List> allPidsList = ReharvestUtils.findPidByType(iterationMap, client, ReharvestItem.fromJSON(itemObject)); - // check size; if size > 10000 - fail state - if (allPidsList.size() < maxItemsToDelete) { - - String proxyURl = proxyMap.get("url"); - if (proxyURl != null) { - ReharvestItem reharvestItem = ReharvestItem.fromJSON(itemObject); - - Map configurations = libraryConfigurations(client, proxyURl,reharvestItem); - // check channels before delete - ReharvestUtils.checkChannelsBeforeDelete(client, configurations); - // delete all asociated pids from index - ReharvestUtils.deleteAllGivenPids(client, destinationMap, allPidsList, onlyShowConfiguration); - // reindex pids - ReharvestUtils.reharvestPIDFromGivenCollections(pid, configurations, ""+onlyShowConfiguration, destinationMap, iterationMap, reharvestItem); - - if (!onlyShowConfiguration) { - changeState(client, wurl, id,"closed"); + List> allPidsList = new ArrayList<>(); + try { + allPidsList = ReharvestUtils.findPidByType(iterationMap, client, ReharvestItem.fromJSON(itemObject), maxItemsToDelete); + // check size; if size > 10000 - fail state + if (allPidsList.size() < maxItemsToDelete) { + + String proxyURl = proxyMap.get("url"); + if (proxyURl != null) { + ReharvestItem reharvestItem = ReharvestItem.fromJSON(itemObject); + + Map configurations = libraryConfigurations(client, proxyURl,reharvestItem); + // check channels before delete + ReharvestUtils.checkChannelsBeforeDelete(client, configurations); + // delete all asociated pids from index + ReharvestUtils.deleteAllGivenPids(client, destinationMap, allPidsList, onlyShowConfiguration); + // reindex pids + ReharvestUtils.reharvestPIDFromGivenCollections(pid, configurations, ""+onlyShowConfiguration, destinationMap, iterationMap, reharvestItem); + + if (!onlyShowConfiguration) { + changeState(client, wurl, id,"closed"); + } + } else { + LOGGER.severe("No proxy configuration"); } } else { - LOGGER.severe("No proxy configuration"); + changeState(client, wurl, id,"too_big"); + String compare = String.format("delete.size() %d >= maxItemstoDelete %d", allPidsList.size(), maxItemsToDelete); + LOGGER.severe(String.format( "Too big to reharvest (%s) -> manual change ", compare)); } - } else { + + } catch(TooBigException ex) { changeState(client, wurl, id,"too_big"); - String compare = String.format("delete.size() %d >= maxItemstoDelete %d", allPidsList.size(), maxItemsToDelete); + String compare = String.format("delete.size() %d >= maxItemstoDelete %d", ex.getCounter(), maxItemsToDelete); LOGGER.severe(String.format( "Too big to reharvest (%s) -> manual change ", compare)); } } else if (topItemFrom.getStatus() == ClientResponse.Status.NOT_FOUND.getStatusCode()) { @@ -130,6 +139,7 @@ public static void main(String[] args) { LOGGER.severe("No proxy or reharvest configuration"); } + } catch (Throwable thr) { if (idReference.get() != null) { Client client = buildClient(); @@ -139,7 +149,8 @@ public static void main(String[] args) { } changeState(client, wurl, idReference.get(),"failed"); } - LOGGER.severe("Exception ex" + thr.getMessage()); + thr.printStackTrace(); + LOGGER.severe("Exception :" + thr.getMessage()); } } diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/TooBigException.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/TooBigException.java new file mode 100644 index 0000000000..aca5c5ebaf --- /dev/null +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/TooBigException.java @@ -0,0 +1,16 @@ +package cz.incad.kramerius; + +/** Too big exception */ +public class TooBigException extends RuntimeException{ + + private int counter; + + public TooBigException(String message, int counter) { + super(message); + this.counter = counter; + } + + public int getCounter() { + return counter; + } +} diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index a68e1239c4..dbe8849506 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -41,9 +41,16 @@ import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.KubernetesReharvestProcess; +import cz.incad.kramerius.TooBigException; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; import cz.incad.kramerius.service.MigrateSolrIndexException; import cz.incad.kramerius.services.ParallelProcessImpl; +import cz.incad.kramerius.services.iterators.IterationItem; +import cz.incad.kramerius.services.iterators.ProcessIterator; +import cz.incad.kramerius.services.iterators.solr.SolrCursorIterator; +import cz.incad.kramerius.services.iterators.solr.SolrFilterQueryIterator; +import cz.incad.kramerius.services.iterators.solr.SolrPageIterator; +import cz.incad.kramerius.services.iterators.solr.SolrIteratorFactory.TypeOfIteration; import cz.incad.kramerius.services.utils.SolrUtils; public class ReharvestUtils { @@ -90,82 +97,66 @@ public static void deleteAllGivenPids(Client client,Map destinat } /** Find all pids by given root.pid */ - public static List> findPidByType(Map iterationMap, Client client, ReharvestItem item) { - LinkedHashSet> allPids = new LinkedHashSet<>(); - String iterationUrl = iterationMap.get("url"); - - String masterQuery = "*:*"; - String filterQuery = "none"; - switch(item.getTypeOfReharvest()) { - case root: - filterQuery = "root.pid:\"" + item.getRootPid() + "\""; - break; - case children: - String ownPidPath = item.getOwnPidPath(); - if (!StringUtils.isAnyString(ownPidPath)) { - throw new IllegalArgumentException("own pid path is null string"); - } - //ownPidPath = ownPidPath.replaceAll(":", "\\\\:")+"/*"; - ownPidPath = ownPidPath.replaceAll(":", "\\\\:")+"*"; - filterQuery = String.format("own_pid_path:%s", ownPidPath);; - break; - } - - + public static List> findPidByType(Map iterationMap, Client client, ReharvestItem item, int maxItems) { try { - String cursorMark = null; - String queryCursorMark = null; - do { - - String sRows = iterationMap.containsKey("rows") ? iterationMap.get("rows") : ITERATION_ROWS_STRING_VALUE ; - LOGGER.info( String.format("Number of iteration rows %s", sRows)); - Element element = pidsCursorQuery(client, iterationUrl, masterQuery, cursorMark, Integer.parseInt(sRows), - filterQuery, "select", "compositeId+pid", "compositeId asc", "", ""); - cursorMark = findCursorMark(element); - queryCursorMark = findQueryCursorMark(element); - - List docs = XMLUtils.getElementsRecursive(element, new XMLUtils.ElementsFilter() { - - @Override - public boolean acceptElement(Element element) { - return element.getNodeName().equals("doc"); + String iterationUrl = iterationMap.get("url"); + String masterQuery = "*:*"; + String filterQuery = "none"; + switch(item.getTypeOfReharvest()) { + case root: + filterQuery = "root.pid:\"" + item.getRootPid() + "\""; + break; + case children: + String ownPidPath = item.getOwnPidPath(); + if (!StringUtils.isAnyString(ownPidPath)) { + throw new IllegalArgumentException("own pid path is null string"); } - }); - - List> pairs = docs.stream().map(doc-> { - - Element pidElm = XMLUtils.findElement(doc, new XMLUtils.ElementsFilter() { - - @Override - public boolean acceptElement(Element element) { - String name = element.getAttribute("name"); - return (name != null && name.equals("pid")); - } - }); - - Element compositeIdElm = XMLUtils.findElement(doc, new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - String name = element.getAttribute("name"); - return (name != null && name.equals("compositeId")); - } - }); - - - if (pidElm != null && compositeIdElm != null ) { - return Pair.of(pidElm.getTextContent(), compositeIdElm.getTextContent()); - } else return null; - }).filter(x -> x != null).collect(Collectors.toList()); - - allPids.addAll(pairs); - LOGGER.info(String.format( "Collected pids to delete %d", allPids.size())); + //ownPidPath = ownPidPath.replaceAll(":", "\\\\:")+"/*"; + ownPidPath = ownPidPath.replaceAll(":", "\\\\:")+"*"; + filterQuery = String.format("own_pid_path:%s", ownPidPath);; + break; + } + + String sRows = iterationMap.containsKey("rows") ? iterationMap.get("rows") : ITERATION_ROWS_STRING_VALUE ; + TypeOfIteration typeOfIteration = iterationMap.containsKey("type") ? TypeOfIteration.valueOf(iterationMap.get("type")) : TypeOfIteration.CURSOR; + ProcessIterator processIterator = null; + switch (typeOfIteration) { + case CURSOR: { + processIterator = new SolrCursorIterator(iterationUrl, masterQuery, filterQuery, "select", "compositeId", "compositeId asc",Integer.parseInt(sRows)); + break; + } + case FILTER: { + processIterator = new SolrFilterQueryIterator( iterationUrl, masterQuery, filterQuery, "select", "compositeId", "compositeId asc",Integer.parseInt(sRows)); + break; + } + case PAGINATION: { + processIterator = new SolrPageIterator( iterationUrl, masterQuery, filterQuery, "select", "compositeId", "compositeId asc",Integer.parseInt(sRows)); + break; + } + } + LOGGER.info(String.format("Solr iterator %s", processIterator.getClass().getName())); + final LinkedHashSet> retvals = new LinkedHashSet<>(); + processIterator.iterate(client, (list)-> { + for (IterationItem it : list) { + //LOGGER.info("Iterating item "+it.toString()); + String compositeId = it.getPid(); + String[] arr = compositeId.split("!"); + if (arr.length == 2) { + Pair pair = Pair.of(arr[1], compositeId); + retvals.add(pair); + } + } + if (retvals.size()>maxItems) { + throw new TooBigException("too_big",retvals.size()); + } - } while ((cursorMark != null && queryCursorMark != null) - && !cursorMark.equals(queryCursorMark)); - } catch (ParserConfigurationException | SAXException | IOException e) { - throw new RuntimeException(e); - } - return new ArrayList>(allPids); + }, ()->{}); + + return new ArrayList>(retvals); + + } catch (TooBigException e) { + throw e; // Vyhození výjimky dál + } } /** Render template */ @@ -227,6 +218,8 @@ public static void checkChannelsBeforeDelete(Client client, Map collections = XMLUtils.getElements(cdkCollection).stream().map(Element::getTextContent).collect(Collectors.toList()); JSONArray jsonArr = new JSONArray(); collections.forEach(jsonArr::put); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java index 28176e0349..06592dc56b 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java @@ -408,6 +408,29 @@ public Response getMetadataMods(@PathParam("pid") String pid) { throw new InternalErrorException(e.getMessage()); } } + + // + @GET + @Path("{pid}/metadata/deletetrig") + @Produces(MediaType.APPLICATION_XML + ";charset=utf-8") + public Response deleteTrigger(@PathParam("pid") String pid) { + try { + // redirect + checkSupportedObjectPid(pid); + ProxyItemHandler redirectHandler = findRedirectHandler(pid, null); + if (redirectHandler != null) { + redirectHandler.deleteTriggeToReharvest(pid); + return Response.ok().build(); + } else { + return Response.status(Response.Status.BAD_REQUEST).build(); + } + } catch (WebApplicationException e) { + throw e; + } catch (Throwable e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new InternalErrorException(e.getMessage()); + } + } @GET diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index a41768b776..4ceb80d564 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -158,7 +158,7 @@ public void write(OutputStream output) throws IOException, WebApplicationExcepti } } - protected void deleteTriggeToReharvest(String pid) { + public void deleteTriggeToReharvest(String pid) { if (reharvestManager != null && pid != null) { try { Document solrDataByPid = this.solrAccess.getSolrDataByPid(pid); From c77e55ac303367ef4139f43ab392345760207bb5 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 5 Jul 2024 17:09:10 +0200 Subject: [PATCH 167/235] Debug trigger --- .../kramerius/rest/apiNew/client/v60/ItemsResource.java | 8 ++++++-- .../client/v60/redirection/ProxyHandlerSupport.java | 5 ++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java index 06592dc56b..6c1f434d37 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java @@ -19,6 +19,8 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.json.JSONObject; + import com.google.inject.Inject; import com.google.inject.Provider; import com.sun.jersey.api.client.Client; @@ -412,15 +414,17 @@ public Response getMetadataMods(@PathParam("pid") String pid) { // @GET @Path("{pid}/metadata/deletetrig") - @Produces(MediaType.APPLICATION_XML + ";charset=utf-8") + @Produces(MediaType.APPLICATION_JSON) public Response deleteTrigger(@PathParam("pid") String pid) { try { // redirect checkSupportedObjectPid(pid); ProxyItemHandler redirectHandler = findRedirectHandler(pid, null); if (redirectHandler != null) { + JSONObject obj = new JSONObject(); + obj.put("msg",String.format("Delete trigger for %s", pid)); redirectHandler.deleteTriggeToReharvest(pid); - return Response.ok().build(); + return Response.ok(obj.toString()).type(MediaType.APPLICATION_JSON).build(); } else { return Response.status(Response.Status.BAD_REQUEST).build(); } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 4ceb80d564..7e9651c9eb 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -160,6 +160,7 @@ public void write(OutputStream output) throws IOException, WebApplicationExcepti public void deleteTriggeToReharvest(String pid) { if (reharvestManager != null && pid != null) { + try { Document solrDataByPid = this.solrAccess.getSolrDataByPid(pid); Element rootPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { @@ -206,9 +207,11 @@ public boolean acceptElement(Element element) { } try { ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", ownParentPid.getTextContent().trim(), pidPath); - // children reharvestItem.setTypeOfReharvest(TypeOfReharvset.children); reharvestItem.setState("waiting_for_approve"); + + LOGGER.info(String.format("Registering item %s", reharvestItem.toJSON().toString()); + this.reharvestManager.register(reharvestItem); } catch (DOMException e) { LOGGER.log(Level.SEVERE,e.getMessage(),e); From 40de83285f46f65d679e2c08e87021fa813b8d2f Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 5 Jul 2024 17:10:09 +0200 Subject: [PATCH 168/235] Debug - delete trigger --- .../rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 7e9651c9eb..34b74017fb 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -210,7 +210,7 @@ public boolean acceptElement(Element element) { reharvestItem.setTypeOfReharvest(TypeOfReharvset.children); reharvestItem.setState("waiting_for_approve"); - LOGGER.info(String.format("Registering item %s", reharvestItem.toJSON().toString()); + LOGGER.info(String.format("Registering item %s", reharvestItem.toJSON().toString())); this.reharvestManager.register(reharvestItem); } catch (DOMException e) { From e3f65f5211264c904da569ea01bdae9c0791a932 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 5 Jul 2024 17:44:04 +0200 Subject: [PATCH 169/235] Support --- .../rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 34b74017fb..58dc61e751 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -160,7 +160,7 @@ public void write(OutputStream output) throws IOException, WebApplicationExcepti public void deleteTriggeToReharvest(String pid) { if (reharvestManager != null && pid != null) { - + LOGGER.info(String.format("Registering pid %s",pid)); try { Document solrDataByPid = this.solrAccess.getSolrDataByPid(pid); Element rootPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { From 83c2bd4a2d10d9dfb17d3004970712d36ce95568 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 5 Jul 2024 18:30:13 +0200 Subject: [PATCH 170/235] CDKCollections in delete trigger --- .../v60/redirection/ProxyHandlerSupport.java | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 58dc61e751..a8ec5541ed 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -1,5 +1,9 @@ package cz.incad.kramerius.rest.apiNew.client.v60.redirection; +import static cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem.LIBRARIES_KEYWORD; +import static cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem.OWN_PID_PATH; +import static cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem.ROOT_PID; + import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -25,6 +29,7 @@ import javax.ws.rs.core.StreamingOutput; import org.apache.commons.io.IOUtils; +import org.json.JSONArray; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -197,6 +202,19 @@ public boolean acceptElement(Element element) { }); + Element cdkCollection = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("arr")) { + String fieldName = element.getAttribute("name"); + return fieldName.equals("cdk.collection"); + } + return false; + } + }); + + + if (rootPid != null && ownPidPath != null && ownParentPid != null) { String pidPath = ownPidPath.getTextContent().trim(); @@ -209,6 +227,12 @@ public boolean acceptElement(Element element) { ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", ownParentPid.getTextContent().trim(), pidPath); reharvestItem.setTypeOfReharvest(TypeOfReharvset.children); reharvestItem.setState("waiting_for_approve"); + + if (cdkCollection != null) { + List collections = XMLUtils.getElements(cdkCollection).stream().map(Element::getTextContent).collect(Collectors.toList()); + reharvestItem.setLibraries(collections); + } + LOGGER.info(String.format("Registering item %s", reharvestItem.toJSON().toString())); From 69a80c7bbf1a83fd0c4ed4e14415a6c837fadb95 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 5 Jul 2024 18:34:38 +0200 Subject: [PATCH 171/235] Removed problematic repo --- build.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.gradle b/build.gradle index a3bc5f0235..df1ae468f1 100644 --- a/build.gradle +++ b/build.gradle @@ -159,10 +159,10 @@ allprojects { mavenCentral() - + /* maven { url "https://m2.duraspace.org/content/repositories/releases" - } + }*/ maven { url "https://maven.java.net/content/repositories/snapshots/" From 6f1ed849a9ac4c8cc8a054ab202b5679138b4300 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 5 Jul 2024 18:47:36 +0200 Subject: [PATCH 172/235] ItemResource - disable debug --- .../rest/apiNew/client/v60/ItemsResource.java | 46 +++++++++---------- .../v60/redirection/ProxyHandlerSupport.java | 4 +- 2 files changed, 25 insertions(+), 25 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java index 6c1f434d37..98b9d522e0 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java @@ -412,29 +412,29 @@ public Response getMetadataMods(@PathParam("pid") String pid) { } // - @GET - @Path("{pid}/metadata/deletetrig") - @Produces(MediaType.APPLICATION_JSON) - public Response deleteTrigger(@PathParam("pid") String pid) { - try { - // redirect - checkSupportedObjectPid(pid); - ProxyItemHandler redirectHandler = findRedirectHandler(pid, null); - if (redirectHandler != null) { - JSONObject obj = new JSONObject(); - obj.put("msg",String.format("Delete trigger for %s", pid)); - redirectHandler.deleteTriggeToReharvest(pid); - return Response.ok(obj.toString()).type(MediaType.APPLICATION_JSON).build(); - } else { - return Response.status(Response.Status.BAD_REQUEST).build(); - } - } catch (WebApplicationException e) { - throw e; - } catch (Throwable e) { - LOGGER.log(Level.SEVERE, e.getMessage(), e); - throw new InternalErrorException(e.getMessage()); - } - } +// @GET +// @Path("{pid}/metadata/deletetrig") +// @Produces(MediaType.APPLICATION_JSON) +// public Response deleteTrigger(@PathParam("pid") String pid) { +// try { +// // redirect +// checkSupportedObjectPid(pid); +// ProxyItemHandler redirectHandler = findRedirectHandler(pid, null); +// if (redirectHandler != null) { +// JSONObject obj = new JSONObject(); +// obj.put("msg",String.format("Delete trigger for %s", pid)); +// redirectHandler.deleteTriggeToReharvest(pid); +// return Response.ok(obj.toString()).type(MediaType.APPLICATION_JSON).build(); +// } else { +// return Response.status(Response.Status.BAD_REQUEST).build(); +// } +// } catch (WebApplicationException e) { +// throw e; +// } catch (Throwable e) { +// LOGGER.log(Level.SEVERE, e.getMessage(), e); +// throw new InternalErrorException(e.getMessage()); +// } +// } @GET diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index a8ec5541ed..f5a86580e8 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -165,7 +165,7 @@ public void write(OutputStream output) throws IOException, WebApplicationExcepti public void deleteTriggeToReharvest(String pid) { if (reharvestManager != null && pid != null) { - LOGGER.info(String.format("Registering pid %s",pid)); + //LOGGER.info(String.format("Registering pid %s",pid)); try { Document solrDataByPid = this.solrAccess.getSolrDataByPid(pid); Element rootPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { @@ -234,7 +234,7 @@ public boolean acceptElement(Element element) { } - LOGGER.info(String.format("Registering item %s", reharvestItem.toJSON().toString())); + //LOGGER.info(String.format("Registering item %s", reharvestItem.toJSON().toString())); this.reharvestManager.register(reharvestItem); } catch (DOMException e) { From 8f567ff4e8f463aeccf59f71c21322f136860c87 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 8 Jul 2024 08:20:29 +0200 Subject: [PATCH 173/235] Channels checks --- .../kramerius/KubernetesReharvestProcess.java | 3 +- .../incad/kramerius/utils/ReharvestUtils.java | 41 +------------- .../v10/proxy/ConnectedInfoResource.java | 53 +++++++++++++++++++ .../kramerius/rest/oai/MetadataExport.java | 1 - 4 files changed, 57 insertions(+), 41 deletions(-) diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java index 2aec02e878..09756b8e24 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -33,6 +33,7 @@ import com.sun.jersey.api.json.JSONConfiguration; import antlr.StringUtils; +import cz.incad.kramerius.cdk.ChannelUtils; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; import cz.incad.kramerius.service.MigrateSolrIndexException; import cz.incad.kramerius.services.utils.kubernetes.KubernetesEnvSupport; @@ -109,7 +110,7 @@ public static void main(String[] args) { Map configurations = libraryConfigurations(client, proxyURl,reharvestItem); // check channels before delete - ReharvestUtils.checkChannelsBeforeDelete(client, configurations); + ChannelUtils.checkSolrChannelEndpoints(client, configurations); // delete all asociated pids from index ReharvestUtils.deleteAllGivenPids(client, destinationMap, allPidsList, onlyShowConfiguration); // reindex pids diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index dbe8849506..843580458c 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -21,7 +21,6 @@ import java.util.logging.Logger; import java.util.stream.Collectors; -import javax.ws.rs.core.MediaType; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; @@ -37,11 +36,10 @@ import com.google.common.io.Files; import com.sun.jersey.api.client.Client; -import com.sun.jersey.api.client.ClientResponse; -import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.KubernetesReharvestProcess; import cz.incad.kramerius.TooBigException; +import cz.incad.kramerius.cdk.ChannelUtils; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; import cz.incad.kramerius.service.MigrateSolrIndexException; import cz.incad.kramerius.services.ParallelProcessImpl; @@ -207,28 +205,6 @@ public static String fq(String api, String pid, ReharvestItem item) throws Unsup return "root_pid:\""+pid+"\""; } - public static void checkChannelsBeforeDelete(Client client, Map collectionConfigurations) { - for (String ac : collectionConfigurations.keySet()) { - JSONObject colObject = collectionConfigurations.get(ac); - String apiVersion = colObject.optString("api","v5"); - if (!colObject.has("forwardurl")) { - LOGGER.severe(String.format("Skipping %s", ac)); - continue; - } - String channel = colObject.optString("forwardurl"); - String fullChannelUrl = solrChannelUrl(apiVersion, channel); - - LOGGER.info(String.format("Checking %s", fullChannelUrl)); - - WebResource configResource = client.resource(fullChannelUrl+"/select?q=*&rows=0&wt=json"); - ClientResponse configReourceStatus = configResource.accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); - if (configReourceStatus.getStatus() == ClientResponse.Status.OK.getStatusCode()) { - // ok - live channel - } else throw new IllegalStateException(String.format("Channel for %s(%s) doesnt work ", ac, channel)); - } - } - public static void reharvestPIDFromGivenCollections(String pid, Map collectionConfigurations, String onlyShowConfiguration, Map destinationMap, Map iterationMap, ReharvestItem item) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchMethodException, MigrateSolrIndexException, IOException, ParserConfigurationException, SAXException { List harvestFiles = new ArrayList<>(); for (String ac : collectionConfigurations.keySet()) { @@ -243,7 +219,7 @@ public static void reharvestPIDFromGivenCollections(String pid, Map iteration = new HashMap<>(iterationMap); String channel = colObject.optString("forwardurl"); - String fullChannelUrl = solrChannelUrl(apiVersion, channel); + String fullChannelUrl = ChannelUtils.solrChannelUrl(apiVersion, channel); iteration.put("url", fullChannelUrl); @@ -283,19 +259,6 @@ public static void reharvestPIDFromGivenCollections(String pid, Map sources = CDKUtils.findSources(solrDataByPid.getDocumentElement()); return leader != null ? leader : (!sources.isEmpty() ? sources.get(0) : null); From 41f30dfe29572cb36810812ab4b00ee786ccae13 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 8 Jul 2024 21:41:42 +0200 Subject: [PATCH 174/235] Connected libraries -> channels --- .../v10/proxy/ConnectedInfoResource.java | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java index 384ebede43..4516565daa 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java @@ -61,11 +61,17 @@ public ConnectedInfoResource() { @GET @Produces(MediaType.APPLICATION_JSON) - public Response getAllConnected() { + public Response getAllConnected(@QueryParam("health")String health) { JSONObject retval = new JSONObject(); this.libraries.allInstances().forEach(library -> { JSONObject json = libraryJSON(library); retval.put(library.getName(), json); + + if (health != null && "true".equals("health")) { + JSONObject channel = new JSONObject(); + channelHealth(library.getName(), channel); + json.put("channel", channel); + } }); return Response.ok(retval).build(); } @@ -196,7 +202,12 @@ public Response getChannelHealth(@PathParam("library") String library) { JSONObject channelObject = new JSONObject(); healthObject.put("channel", channelObject); - + channelHealth(library, channelObject); + + return Response.ok(healthObject.toString()).build(); + } + + private void channelHealth(String library, JSONObject channelObject) { String baseurl = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".baseurl"); String api = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".api"); boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + library + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + library + ".licenses") : false; @@ -204,7 +215,7 @@ public Response getChannelHealth(@PathParam("library") String library) { if (channelAccess) { OneInstance inst = this.libraries.find(library); - healthObject.put("status", inst.isConnected()); + //healthObject.put("status", inst.isConnected()); if (inst.isConnected()) { // solr try { @@ -227,8 +238,6 @@ public Response getChannelHealth(@PathParam("library") String library) { } } } - - return Response.ok(healthObject.toString()).build(); } From 0b0ddce58b4166d06cf01f9890a885d0e9942b90 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 8 Jul 2024 21:52:40 +0200 Subject: [PATCH 175/235] CahnnelUtils --- .../cz/incad/kramerius/cdk/ChannelUtils.java | 116 ++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java diff --git a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java new file mode 100644 index 0000000000..3e647a1f08 --- /dev/null +++ b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java @@ -0,0 +1,116 @@ +package cz.incad.kramerius.cdk; + +import java.util.Map; +import java.util.logging.Logger; + +import javax.ws.rs.core.MediaType; + +import org.json.JSONObject; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.ClientResponse.Status; +import com.sun.jersey.api.client.WebResource; + + +public class ChannelUtils { + + public static Logger LOGGER = Logger.getLogger(ChannelUtils.class.getName()); + + private ChannelUtils() { + } + public static String userChannelUrl(String apiVersion, String channel) { + String fullChannelUrl = null; + if (apiVersion.toLowerCase().equals("v5")) { + //channel = + fullChannelUrl = channel+(channel.endsWith("/") ? "" : "/")+"api/v5.0/cdk/forward/user"; + // channel+(channel.endsWith("/") ? "" : "/")+"api/v5.0/cdk/forward/sync/solr"); + } else { + fullChannelUrl = channel+(channel.endsWith("/") ? "" : "/")+"api/cdk/v7.0/forward/user"; + //channel+(channel.endsWith("/") ? "" : "/")+"api/cdk/v7.0/forward/sync/solr"); + } + return fullChannelUrl; + } + + + public static String solrChannelUrl(String apiVersion, String channel) { + String fullChannelUrl = null; + if (apiVersion.toLowerCase().equals("v5")) { + //channel = + fullChannelUrl = channel+(channel.endsWith("/") ? "" : "/")+"api/v5.0/cdk/forward/sync/solr"; + // channel+(channel.endsWith("/") ? "" : "/")+"api/v5.0/cdk/forward/sync/solr"); + } else { + fullChannelUrl = channel+(channel.endsWith("/") ? "" : "/")+"api/cdk/v7.0/forward/sync/solr"; + //channel+(channel.endsWith("/") ? "" : "/")+"api/cdk/v7.0/forward/sync/solr"); + } + return fullChannelUrl; + } + + public static void userChannelEndpoints(Client client, Map collectionConfigurations) { + for (String ac : collectionConfigurations.keySet()) { + JSONObject colObject = collectionConfigurations.get(ac); + String apiVersion = colObject.optString("api","v5"); + if (!colObject.has("forwardurl")) { + LOGGER.severe(String.format("Skipping %s", ac)); + continue; + } + String channel = colObject.optString("forwardurl"); + String fullChannelUrl = userChannelUrl(apiVersion, channel); + + LOGGER.info(String.format("Checking %s", fullChannelUrl)); + checkUserChannelEndpoint(client, ac, fullChannelUrl); + +// WebResource configResource = client.resource(fullChannelUrl); +// ClientResponse configReourceStatus = configResource.accept(MediaType.APPLICATION_JSON) +// .get(ClientResponse.class); +// if (configReourceStatus.getStatus() == ClientResponse.Status.OK.getStatusCode()) { +// // ok - live channel +// } else throw new IllegalStateException(String.format("Channel for %s(%s) doesnt work ", ac, channel)); + } + } + + public static void checkUserChannelEndpoint(Client client,String ac,String fullChannelUrl) { + WebResource configResource = client.resource(fullChannelUrl); + ClientResponse configReourceStatus = configResource.accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + if (configReourceStatus.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + // ok - live channel + } else throw new IllegalStateException(String.format("Channel for %s(%s) doesnt work ", ac, fullChannelUrl)); + + } + + public static void checkSolrChannelEndpoints(Client client, Map collectionConfigurations) { + for (String ac : collectionConfigurations.keySet()) { + JSONObject colObject = collectionConfigurations.get(ac); + String apiVersion = colObject.optString("api","v5"); + if (!colObject.has("forwardurl")) { + LOGGER.severe(String.format("Skipping %s", ac)); + continue; + } + String channel = colObject.optString("forwardurl"); + String fullChannelUrl = solrChannelUrl(apiVersion, channel); + + LOGGER.info(String.format("Checking %s", fullChannelUrl)); + checkSolrChannelEndpoint(client, ac,fullChannelUrl); + +// WebResource configResource = client.resource(fullChannelUrl+"/select?q=*&rows=0&wt=json"); +// ClientResponse configReourceStatus = configResource.accept(MediaType.APPLICATION_JSON) +// .get(ClientResponse.class); +// if (configReourceStatus.getStatus() == ClientResponse.Status.OK.getStatusCode()) { +// // ok - live channel +// } else throw new IllegalStateException(String.format("Channel for %s(%s) doesnt work ", ac, channel)); + } + } + + + public static void checkSolrChannelEndpoint(Client client, String ac,String fullChannelUrl) { + WebResource configResource = client.resource(fullChannelUrl+"/select?q=*&rows=0&wt=json"); + ClientResponse configReourceStatus = configResource.accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + if (configReourceStatus.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + // ok - live channel + } else throw new IllegalStateException(String.format("Channel for %s(%s) doesnt work ", ac, fullChannelUrl)); + } + +} + From e45b17915ae44109dce3158d2154c4c325f860a4 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 8 Jul 2024 22:26:00 +0200 Subject: [PATCH 176/235] Connected; info messages --- .../rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java index 4516565daa..72cfc8964a 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java @@ -67,10 +67,15 @@ public Response getAllConnected(@QueryParam("health")String health) { JSONObject json = libraryJSON(library); retval.put(library.getName(), json); + LOGGER.info(String.format("Parameter health '%s'", health)); + if (health != null && "true".equals("health")) { JSONObject channel = new JSONObject(); channelHealth(library.getName(), channel); json.put("channel", channel); + + LOGGER.info(String.format("Channel json is '%s'", channel.toString())); + } }); return Response.ok(retval).build(); From c3e1de1228f15a0f32ff47ebec5a770877d366fb Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 8 Jul 2024 22:50:56 +0200 Subject: [PATCH 177/235] Debug info --- .../rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java index 72cfc8964a..e0eda4895a 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java @@ -67,15 +67,14 @@ public Response getAllConnected(@QueryParam("health")String health) { JSONObject json = libraryJSON(library); retval.put(library.getName(), json); - LOGGER.info(String.format("Parameter health '%s'", health)); + boolean healthCheck = Boolean.valueOf(health); - if (health != null && "true".equals("health")) { + LOGGER.info(String.format("Parameter health '%b'", healthCheck)); + if (healthCheck) { JSONObject channel = new JSONObject(); channelHealth(library.getName(), channel); json.put("channel", channel); - LOGGER.info(String.format("Channel json is '%s'", channel.toString())); - } }); return Response.ok(retval).build(); From 493804f8e029ccbbde61c58000771e43f03e6762 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 8 Jul 2024 23:15:00 +0200 Subject: [PATCH 178/235] Cahnged endpoint - configuration /config/channel/health --- .../admin/v10/proxy/ConnectedInfoResource.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java index e0eda4895a..adf9e55ba6 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java @@ -39,6 +39,7 @@ import cz.incad.kramerius.timestamps.Timestamp; import cz.incad.kramerius.timestamps.TimestampStore; import cz.incad.kramerius.timestamps.impl.SolrTimestamp; +import cz.incad.kramerius.utils.StringUtils; import cz.incad.kramerius.utils.conf.KConfiguration; @Path("/admin/v7.0/connected") @@ -61,21 +62,21 @@ public ConnectedInfoResource() { @GET @Produces(MediaType.APPLICATION_JSON) - public Response getAllConnected(@QueryParam("health")String health) { + public Response getAllConnected(/*@QueryParam("health")String health*/) { JSONObject retval = new JSONObject(); this.libraries.allInstances().forEach(library -> { JSONObject json = libraryJSON(library); retval.put(library.getName(), json); + /* boolean healthCheck = Boolean.valueOf(health); - LOGGER.info(String.format("Parameter health '%b'", healthCheck)); if (healthCheck) { JSONObject channel = new JSONObject(); channelHealth(library.getName(), channel); json.put("channel", channel); LOGGER.info(String.format("Channel json is '%s'", channel.toString())); - } + }*/ }); return Response.ok(retval).build(); } @@ -220,7 +221,9 @@ private void channelHealth(String library, JSONObject channelObject) { if (channelAccess) { OneInstance inst = this.libraries.find(library); //healthObject.put("status", inst.isConnected()); - if (inst.isConnected()) { + + if (inst.isConnected() && StringUtils.isAnyString(channel)) { + channelObject.put("enabled", true); // solr try { String solrChannelUrl = ChannelUtils.solrChannelUrl(api, channel); @@ -229,6 +232,7 @@ private void channelHealth(String library, JSONObject channelObject) { } catch (Exception e) { LOGGER.log(Level.SEVERE,e.getMessage(),e); channelObject.put("solr", false); + channelObject.put("solr_message", e.getMessage()); } // user @@ -239,7 +243,10 @@ private void channelHealth(String library, JSONObject channelObject) { } catch (Exception e) { LOGGER.log(Level.SEVERE,e.getMessage(),e); channelObject.put("user", false); + channelObject.put("user_message", e.getMessage()); } + } else { + channelObject.put("enabled", false); } } } From 9562b9142b1497037b5fdf5fb4bffc81c56e1aa2 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 9 Jul 2024 22:59:16 +0200 Subject: [PATCH 179/235] New project --- .../kubernetes/KubernetesEnvSupport.java | 9 +++- .../incad/kramerius/utils/ReharvestUtils.java | 3 ++ processes/cdksync/build.gradle | 35 +++++++++++++++ .../cdk/sync/KubernetesSyncProcess.java | 43 +++++++++++++++++++ .../v10/reharvest/ReharvestResource.java | 37 ++++++++++++---- settings.gradle | 1 + 6 files changed, 118 insertions(+), 10 deletions(-) create mode 100644 processes/cdksync/build.gradle create mode 100644 processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/kubernetes/KubernetesEnvSupport.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/kubernetes/KubernetesEnvSupport.java index 6f8c464349..8ffbfc62b4 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/kubernetes/KubernetesEnvSupport.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/kubernetes/KubernetesEnvSupport.java @@ -13,7 +13,7 @@ public class KubernetesEnvSupport { public static final String ITERATION_PREFIX = "ITERATION_"; public static final String CHECK_PREFIX = "CHECK_"; public static final String DEST_PREFIX = "DESTINATION_"; - + public static final String PROXY_API_URL = "PROXY_API_URL"; public static final String REHARVEST_URL = "REHARVEST_URL"; @@ -21,6 +21,7 @@ public class KubernetesEnvSupport { public static final String TMSP_PREFIX = "TIMESTAMP_"; public static final String PROXY_API_PREFIX = "PROXY_API_"; public static final String REHARVEST_PREFIX = "REHARVEST_"; + public static final String ITERATION_SYNC_PREFIX = "ITERATION_SYNC_"; public static void prefixVariables(Map env, Map templateMap, String prefix) { env.keySet().forEach(key -> { @@ -67,6 +68,12 @@ public static Map iterationMap(Map env) { return iteration; } + public static Map iterationSyncMap(Map env) { + Map iteration = new HashMap<>(); + prefixVariables(env, iteration, ITERATION_SYNC_PREFIX); + return iteration; + } + public static Map proxyMap(Map env) { Map proxy = new HashMap<>(); prefixVariables(env, proxy, PROXY_API_PREFIX); diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index 843580458c..ba4ca96484 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -101,6 +101,7 @@ public static List> findPidByType(Map iterat String masterQuery = "*:*"; String filterQuery = "none"; switch(item.getTypeOfReharvest()) { + case new_root: case root: filterQuery = "root.pid:\"" + item.getRootPid() + "\""; break; @@ -177,6 +178,8 @@ public static String renderTemplate(String api, Map iteration, Ma public static String fq(String api, String pid, ReharvestItem item) throws UnsupportedEncodingException { switch(item.getTypeOfReharvest()) { + + case new_root: case root: switch(api) { case "v7": return "root.pid:\""+pid+"\""; diff --git a/processes/cdksync/build.gradle b/processes/cdksync/build.gradle new file mode 100644 index 0000000000..e2ecd86a26 --- /dev/null +++ b/processes/cdksync/build.gradle @@ -0,0 +1,35 @@ +//apply plugin: 'application' +apply plugin: 'java' + + +dependencies { + api project(':shared:common') + api project(':processes:cdkprocesses') + + implementation project(':processes:solr-migration') + + implementation project(':search-index') + + testImplementation 'org.xmlunit:xmlunit-core:2.8.4' + testImplementation group: 'org.xmlunit', name: 'xmlunit-matchers', version: '2.8.4' + testImplementation group: 'org.xmlunit', name: 'xmlunit-assertj', version: '2.8.4' + testImplementation "org.easymock:easymock:4.3" + +} + + + +configurations.api { + exclude group: 'org.bouncycastle' + exclude group: 'com.lowagie' + exclude group: 'org.apache.pdfbox' + exclude group: 'com.levigo.jbig2' + exclude group: 'org.apache.solr:solr-solrj' + //exclude group: 'com.sun.jersey:jersey-json' +} + + +/* +application { + mainClass = 'cz.incad.kramerius.KubernetesReharvestProcess' +}*/ diff --git a/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java b/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java new file mode 100644 index 0000000000..4041acd27b --- /dev/null +++ b/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java @@ -0,0 +1,43 @@ +package cz.inovatika.cdk.sync; + +import java.util.Map; +import java.util.TimeZone; +import java.util.logging.Logger; + +import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.config.ClientConfig; +import com.sun.jersey.api.client.config.DefaultClientConfig; +import com.sun.jersey.api.json.JSONConfiguration; + +import cz.incad.kramerius.services.utils.kubernetes.KubernetesEnvSupport; + + +public class KubernetesSyncProcess { + + public static final Logger LOGGER = Logger.getLogger(KubernetesSyncProcess.class.getName()); + + public static final String ONLY_SHOW_CONFIGURATION = "ONLY_SHOW_CONFIGURATION"; + + + + protected static Client buildClient() { + // Client client = Client.create(); + ClientConfig cc = new DefaultClientConfig(); + cc.getProperties().put(ClientConfig.PROPERTY_FOLLOW_REDIRECTS, true); + cc.getFeatures().put(JSONConfiguration.FEATURE_POJO_MAPPING, Boolean.TRUE); + return Client.create(cc); + } + + public static void main(String[] args) { + + TimeZone.setDefault(TimeZone.getTimeZone("Europe/Prague")); + + Map env = System.getenv(); + Map iterationMap = KubernetesEnvSupport.iterationMap(env); + if (!iterationMap.containsKey("batch")) { + iterationMap.put("batch", "45"); + } + + + } +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index 1899c16470..4b1c12cec4 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -102,6 +102,7 @@ public Response register(String json) { if (!jsonObj.has(PID_KEYWORD)) { throw new BadRequestException(" Request must contain pid "); } + if (!jsonObj.has(OWN_PID_PATH) || !jsonObj.has(ROOT_PID)) { Document solrDataByPid = this.solrAccess.getSolrDataByPid(jsonObj.getString(PID_KEYWORD)); Element rootPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { @@ -154,15 +155,33 @@ public boolean acceptElement(Element element) { // nasel to v indexu (pro polozky, ktere nejsou v indexu je potreba novy typ harvestu ReharvestItem item = ReharvestItem.fromJSON(jsonObj); - // check - if (item != null && StringUtils.isAnyString(item.getRootPid()) && StringUtils.isAnyString(item.getOwnPidPath())) { - item.setTimestamp(Instant.now()); - this.reharvestManager.register(item); - return Response.ok(item.toJSON().toString()).build(); - } else { - JSONObject errorObject = new JSONObject(); - errorObject.put("error", "No root pid or own_pid_path"); - return Response.status(Response.Status.BAD_REQUEST).entity(errorObject.toString()).type(MediaType.APPLICATION_JSON_TYPE).build(); + + switch(item.getTypeOfReharvest()) { + case root: + case children: + + if (item != null && StringUtils.isAnyString(item.getRootPid()) && StringUtils.isAnyString(item.getOwnPidPath())) { + item.setTimestamp(Instant.now()); + this.reharvestManager.register(item); + return Response.ok(item.toJSON().toString()).build(); + } else { + JSONObject errorObject = new JSONObject(); + errorObject.put("error", "No root pid or own_pid_path"); + return Response.status(Response.Status.BAD_REQUEST).entity(errorObject.toString()).type(MediaType.APPLICATION_JSON_TYPE).build(); + } + + case new_root: + if (item != null && StringUtils.isAnyString(item.getRootPid())) { + item.setTimestamp(Instant.now()); + this.reharvestManager.register(item); + return Response.ok(item.toJSON().toString()).build(); + } else { + JSONObject errorObject = new JSONObject(); + errorObject.put("error", "No root pid or own_pid_path"); + return Response.status(Response.Status.BAD_REQUEST).entity(errorObject.toString()).type(MediaType.APPLICATION_JSON_TYPE).build(); + } + default: + throw new IllegalStateException(String.format("Uknown type of reharvest %s", item.getTypeOfReharvest())); } } catch (JSONException | ParseException e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); diff --git a/settings.gradle b/settings.gradle index fa7b184bc4..9534c4d917 100644 --- a/settings.gradle +++ b/settings.gradle @@ -20,6 +20,7 @@ include 'processes:import-jaxb-srwdc' include 'processes:import-mets' include 'processes:cdkprocesses' include 'processes:cdkreharvest' +include 'processes:cdksync' include 'processes:solr-migration' //dependencies From 483336d74b9a3b76bec3b9c413ce20fcdc379298 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 10 Jul 2024 23:26:20 +0200 Subject: [PATCH 180/235] Reharvest resource --- .../rest/apiNew/admin/v10/reharvest/ReharvestItem.java | 3 ++- .../rest/apiNew/admin/v10/reharvest/ReharvestResource.java | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java index 3baeda4b3d..1a48fc857d 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java @@ -223,7 +223,8 @@ public static ReharvestItem fromJSON(JSONObject json) throws ParseException { if (json.has(TYPE_KEYWORD)) { String tt = json.getString(TYPE_KEYWORD); - item.setTypeOfReharvest(TypeOfReharvset.root.name().equals(tt) ? TypeOfReharvset.root : TypeOfReharvset.children); + TypeOfReharvset typeOfHarvest = TypeOfReharvset.valueOf(tt); + item.setTypeOfReharvest(typeOfHarvest); } if (json.has(LIBRARIES_KEYWORD)) { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index 4b1c12cec4..fe116143b4 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -138,10 +138,10 @@ public boolean acceptElement(Element element) { }); - if (rootPid != null) { + if (rootPid != null && !jsonObj.has(ROOT_PID)) { jsonObj.put(ROOT_PID, rootPid.getTextContent()); } - if (ownPidPath != null) { + if (ownPidPath != null && !jsonObj.has(OWN_PID_PATH)) { jsonObj.put(OWN_PID_PATH, ownPidPath.getTextContent()); } @@ -177,7 +177,7 @@ public boolean acceptElement(Element element) { return Response.ok(item.toJSON().toString()).build(); } else { JSONObject errorObject = new JSONObject(); - errorObject.put("error", "No root pid or own_pid_path"); + errorObject.put("error", "No root pid"); return Response.status(Response.Status.BAD_REQUEST).entity(errorObject.toString()).type(MediaType.APPLICATION_JSON_TYPE).build(); } default: From ca16beff15bf443b05036ca03e417e2df8861650 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sat, 13 Jul 2024 08:05:44 +0200 Subject: [PATCH 181/235] Users check --- .../v10/proxy/ConnectedInfoResource.java | 39 +++++++++++++++++-- .../cz/incad/kramerius/cdk/ChannelUtils.java | 16 ++++++-- 2 files changed, 48 insertions(+), 7 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java index adf9e55ba6..54d3537a81 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java @@ -30,7 +30,10 @@ import cz.incad.kramerius.cdk.ChannelUtils; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance.InstanceType; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance.TypeOfChangedStatus; +import cz.incad.kramerius.rest.apiNew.client.v60.redirection.user.V5ForwardUserHandler; +import cz.incad.kramerius.rest.apiNew.client.v60.redirection.user.V7ForwardUserHandler; import cz.incad.kramerius.rest.apiNew.client.v60.libs.PhysicalLocationMap; import cz.incad.kramerius.rest.apiNew.exceptions.ForbiddenException; import cz.incad.kramerius.rest.apiNew.exceptions.InternalErrorException; @@ -205,16 +208,20 @@ public Response config(@PathParam("library") String library) { public Response getChannelHealth(@PathParam("library") String library) { JSONObject healthObject = new JSONObject(); JSONObject channelObject = new JSONObject(); + JSONObject usersObject = new JSONObject(); healthObject.put("channel", channelObject); - channelHealth(library, channelObject); + healthObject.put("users", usersObject); + channelHealth(library, channelObject,usersObject); + + return Response.ok(healthObject.toString()).build(); } - private void channelHealth(String library, JSONObject channelObject) { + private void channelHealth(String library, JSONObject channelObject, JSONObject usersObject) { String baseurl = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".baseurl"); - String api = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".api"); + String api = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".api","v5"); boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + library + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + library + ".licenses") : false; String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".forwardurl"); @@ -238,8 +245,20 @@ private void channelHealth(String library, JSONObject channelObject) { // user try { String fullChannelUrl = ChannelUtils.userChannelUrl(api, channel); - ChannelUtils.checkUserChannelEndpoint(this.client, library, fullChannelUrl); + + JSONObject notLoggedUser = new JSONObject(); + JSONObject dnntUser = new JSONObject(); + + JSONObject notLoggedJSON = ChannelUtils.checkUserChannelEndpoint(this.client, library, fullChannelUrl, false); channelObject.put("user", true); + + usersDetail(notLoggedUser, notLoggedJSON); + usersObject.put("notLogged", notLoggedUser); + + JSONObject dnntJSON = ChannelUtils.checkUserChannelEndpoint(this.client, library, fullChannelUrl, true); + usersDetail(dnntUser, dnntJSON); + + } catch (Exception e) { LOGGER.log(Level.SEVERE,e.getMessage(),e); channelObject.put("user", false); @@ -251,6 +270,18 @@ private void channelHealth(String library, JSONObject channelObject) { } } + private void usersDetail(JSONObject notLogged, JSONObject userJson) { + JSONArray rolesJSONArray = userJson.optJSONArray("roles"); + notLogged.put("roles", rolesJSONArray); + if (userJson.has("labels")) { + JSONArray licensesArray = userJson.getJSONArray("labels"); + notLogged.put("licenses", licensesArray); + } else if (userJson.has("licenses")){ + JSONArray licensesArray = userJson.getJSONArray("licenses"); + notLogged.put("licenses", licensesArray); + } + } + @PUT @Path("{library}/timestamp") diff --git a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java index 3e647a1f08..47c94e4d86 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java @@ -10,8 +10,12 @@ import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.ClientResponse.Status; + + import com.sun.jersey.api.client.WebResource; +import cz.incad.kramerius.utils.StringUtils; + public class ChannelUtils { @@ -60,6 +64,7 @@ public static void userChannelEndpoints(Client client, Map co LOGGER.info(String.format("Checking %s", fullChannelUrl)); checkUserChannelEndpoint(client, ac, fullChannelUrl); + // WebResource configResource = client.resource(fullChannelUrl); // ClientResponse configReourceStatus = configResource.accept(MediaType.APPLICATION_JSON) // .get(ClientResponse.class); @@ -69,11 +74,16 @@ public static void userChannelEndpoints(Client client, Map co } } - public static void checkUserChannelEndpoint(Client client,String ac,String fullChannelUrl) { + public static JSONObject checkUserChannelEndpoint(Client client, String ac,String fullChannelUrl, boolean header) { WebResource configResource = client.resource(fullChannelUrl); - ClientResponse configReourceStatus = configResource.accept(MediaType.APPLICATION_JSON) + if (header) { + configResource.header("CDK_TOKEN_PARAMETERS", "header_shib-session-id=_dd68cbd66641c9b647b05509ac0241fa|header_shib-session-expires=1592847906|header_shib-identity-provider=https://shibboleth.mzk.cz/simplesaml/metadata.xml|header_shib-authentication-method=urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport|header_shib-handler=https://dnnt.mzk.cz/Shibboleth.sso|header_eppn=test|header_entitlement=cdk.entitlement|header_eduPersonEntitlement=urn:mace:dir:entitlement:common-lib-terms|header_knav_type=validuser|header_knav_session_eppn=test|header_displayName=Inovatika|header_expiration_time=1720851072|header_knav_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_remote_user=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_eduPersonScopedAffiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_token_id=e96c9aec-a262-4fc6-8fe1-4104bdaa8dc8|header_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_knav_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_eduPersonPrincipalName=principalname@lib.cas.cz|header_knav_dnnt_user=test|header_eduPersonUniqueId=eduperson@lib.cas.cz|header_expires_in=1801|header_preffered_user_name=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_email=xxx@time.com|header_authentication_time=1720849271|header_ip_address=xx.xx.xx.xx"); + } + ClientResponse userRes = configResource.accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); - if (configReourceStatus.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + if (userRes.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + String t = userRes.getEntity(String.class); + return new JSONObject(t); // ok - live channel } else throw new IllegalStateException(String.format("Channel for %s(%s) doesnt work ", ac, fullChannelUrl)); From df05e48f33f6efefc50ecf03de283a63586f55b8 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sat, 13 Jul 2024 08:07:48 +0200 Subject: [PATCH 182/235] Dnnt users; health check --- .../rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java | 1 + 1 file changed, 1 insertion(+) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java index 54d3537a81..6a52867202 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java @@ -257,6 +257,7 @@ private void channelHealth(String library, JSONObject channelObject, JSONObject JSONObject dnntJSON = ChannelUtils.checkUserChannelEndpoint(this.client, library, fullChannelUrl, true); usersDetail(dnntUser, dnntJSON); + usersObject.put("dnnt", notLoggedUser); } catch (Exception e) { From 431eebbd6a1d4145b5a2b770bb8e553258d11d96 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sat, 13 Jul 2024 20:42:17 +0200 Subject: [PATCH 183/235] Fix utility method --- .../src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java index 47c94e4d86..70813283d6 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java @@ -62,7 +62,7 @@ public static void userChannelEndpoints(Client client, Map co String fullChannelUrl = userChannelUrl(apiVersion, channel); LOGGER.info(String.format("Checking %s", fullChannelUrl)); - checkUserChannelEndpoint(client, ac, fullChannelUrl); + checkUserChannelEndpoint(client, ac, fullChannelUrl,false); // WebResource configResource = client.resource(fullChannelUrl); From 8ea9b0c7fb11b6daf2c08b4efb653d66c16693db Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 14 Jul 2024 18:23:53 +0200 Subject: [PATCH 184/235] Channel - test check --- .../java/cz/incad/kramerius/cdk/ChannelUtils.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java index 70813283d6..cc673ae3f4 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java @@ -13,6 +13,7 @@ import com.sun.jersey.api.client.WebResource; +import com.sun.jersey.api.client.WebResource.Builder; import cz.incad.kramerius.utils.StringUtils; @@ -65,22 +66,17 @@ public static void userChannelEndpoints(Client client, Map co checkUserChannelEndpoint(client, ac, fullChannelUrl,false); -// WebResource configResource = client.resource(fullChannelUrl); -// ClientResponse configReourceStatus = configResource.accept(MediaType.APPLICATION_JSON) -// .get(ClientResponse.class); -// if (configReourceStatus.getStatus() == ClientResponse.Status.OK.getStatusCode()) { -// // ok - live channel -// } else throw new IllegalStateException(String.format("Channel for %s(%s) doesnt work ", ac, channel)); } } public static JSONObject checkUserChannelEndpoint(Client client, String ac,String fullChannelUrl, boolean header) { WebResource configResource = client.resource(fullChannelUrl); + Builder builder = configResource.accept(MediaType.APPLICATION_JSON); if (header) { - configResource.header("CDK_TOKEN_PARAMETERS", "header_shib-session-id=_dd68cbd66641c9b647b05509ac0241fa|header_shib-session-expires=1592847906|header_shib-identity-provider=https://shibboleth.mzk.cz/simplesaml/metadata.xml|header_shib-authentication-method=urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport|header_shib-handler=https://dnnt.mzk.cz/Shibboleth.sso|header_eppn=test|header_entitlement=cdk.entitlement|header_eduPersonEntitlement=urn:mace:dir:entitlement:common-lib-terms|header_knav_type=validuser|header_knav_session_eppn=test|header_displayName=Inovatika|header_expiration_time=1720851072|header_knav_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_remote_user=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_eduPersonScopedAffiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_token_id=e96c9aec-a262-4fc6-8fe1-4104bdaa8dc8|header_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_knav_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_eduPersonPrincipalName=principalname@lib.cas.cz|header_knav_dnnt_user=test|header_eduPersonUniqueId=eduperson@lib.cas.cz|header_expires_in=1801|header_preffered_user_name=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_email=xxx@time.com|header_authentication_time=1720849271|header_ip_address=xx.xx.xx.xx"); + builder = configResource.header("CDK_TOKEN_PARAMETERS", "header_shib-session-id=_dd68cbd66641c9b647b05509ac0241fa|header_shib-session-expires=1592847906|header_shib-identity-provider=https://shibboleth.mzk.cz/simplesaml/metadata.xml|header_shib-authentication-method=urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport|header_shib-handler=https://dnnt.mzk.cz/Shibboleth.sso|header_eppn=test|header_entitlement=cdk.entitlement|header_eduPersonEntitlement=urn:mace:dir:entitlement:common-lib-terms|header_knav_type=validuser|header_knav_session_eppn=test|header_displayName=Inovatika|header_expiration_time=1720851072|header_knav_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_remote_user=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_eduPersonScopedAffiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_token_id=e96c9aec-a262-4fc6-8fe1-4104bdaa8dc8|header_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_knav_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_eduPersonPrincipalName=principalname@lib.cas.cz|header_knav_dnnt_user=test|header_eduPersonUniqueId=eduperson@lib.cas.cz|header_expires_in=1801|header_preffered_user_name=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_email=xxx@time.com|header_authentication_time=1720849271|header_ip_address=xx.xx.xx.xx"); } - ClientResponse userRes = configResource.accept(MediaType.APPLICATION_JSON) - .get(ClientResponse.class); + + ClientResponse userRes = builder.get(ClientResponse.class); if (userRes.getStatus() == ClientResponse.Status.OK.getStatusCode()) { String t = userRes.getEntity(String.class); return new JSONObject(t); From b07324e1bd6f0ea7d4249e81be9f69c1f547d7f5 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 14 Jul 2024 20:14:43 +0200 Subject: [PATCH 185/235] Change builder --- .../main/java/cz/incad/kramerius/cdk/ChannelUtils.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java index cc673ae3f4..ab88b6b988 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java @@ -71,18 +71,19 @@ public static void userChannelEndpoints(Client client, Map co public static JSONObject checkUserChannelEndpoint(Client client, String ac,String fullChannelUrl, boolean header) { WebResource configResource = client.resource(fullChannelUrl); - Builder builder = configResource.accept(MediaType.APPLICATION_JSON); + Builder builder = null; if (header) { - builder = configResource.header("CDK_TOKEN_PARAMETERS", "header_shib-session-id=_dd68cbd66641c9b647b05509ac0241fa|header_shib-session-expires=1592847906|header_shib-identity-provider=https://shibboleth.mzk.cz/simplesaml/metadata.xml|header_shib-authentication-method=urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport|header_shib-handler=https://dnnt.mzk.cz/Shibboleth.sso|header_eppn=test|header_entitlement=cdk.entitlement|header_eduPersonEntitlement=urn:mace:dir:entitlement:common-lib-terms|header_knav_type=validuser|header_knav_session_eppn=test|header_displayName=Inovatika|header_expiration_time=1720851072|header_knav_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_remote_user=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_eduPersonScopedAffiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_token_id=e96c9aec-a262-4fc6-8fe1-4104bdaa8dc8|header_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_knav_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_eduPersonPrincipalName=principalname@lib.cas.cz|header_knav_dnnt_user=test|header_eduPersonUniqueId=eduperson@lib.cas.cz|header_expires_in=1801|header_preffered_user_name=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_email=xxx@time.com|header_authentication_time=1720849271|header_ip_address=xx.xx.xx.xx"); + builder = configResource.accept(MediaType.APPLICATION_JSON).header("CDK_TOKEN_PARAMETERS", "header_shib-session-id=_dd68cbd66641c9b647b05509ac0241fa|header_shib-session-expires=1592847906|header_shib-identity-provider=https://shibboleth.mzk.cz/simplesaml/metadata.xml|header_shib-authentication-method=urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport|header_shib-handler=https://dnnt.mzk.cz/Shibboleth.sso|header_eppn=test|header_entitlement=cdk.entitlement|header_eduPersonEntitlement=urn:mace:dir:entitlement:common-lib-terms|header_knav_type=validuser|header_knav_session_eppn=test|header_displayName=Inovatika|header_expiration_time=1720851072|header_knav_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_remote_user=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_eduPersonScopedAffiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_token_id=e96c9aec-a262-4fc6-8fe1-4104bdaa8dc8|header_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_knav_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_eduPersonPrincipalName=principalname@lib.cas.cz|header_knav_dnnt_user=test|header_eduPersonUniqueId=eduperson@lib.cas.cz|header_expires_in=1801|header_preffered_user_name=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_email=xxx@time.com|header_authentication_time=1720849271|header_ip_address=xx.xx.xx.xx"); + LOGGER.info("CDK_TOKEN_PARAMETERS = "+configResource+";"); + } else { + builder = configResource.accept(MediaType.APPLICATION_JSON); } - ClientResponse userRes = builder.get(ClientResponse.class); if (userRes.getStatus() == ClientResponse.Status.OK.getStatusCode()) { String t = userRes.getEntity(String.class); return new JSONObject(t); // ok - live channel } else throw new IllegalStateException(String.format("Channel for %s(%s) doesnt work ", ac, fullChannelUrl)); - } public static void checkSolrChannelEndpoints(Client client, Map collectionConfigurations) { From adddf136ab6f2e15f7a5c7a3266fd62760bb600d Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sat, 20 Jul 2024 22:43:03 +0200 Subject: [PATCH 186/235] Process; roles --- .../kubernetes/KubernetesEnvSupport.java | 12 +- .../incad/kramerius/utils/ReharvestUtils.java | 2 +- processes/cdksync/build.gradle | 1 + .../cdk/sync/KubernetesSyncProcess.java | 223 +++++++++++++++++- .../inovatika/cdk/sync/utils/SyncUtils.java | 7 + .../v10/proxy/ConnectedInfoResource.java | 23 +- .../cz/incad/kramerius/cdk/ChannelUtils.java | 5 +- 7 files changed, 258 insertions(+), 15 deletions(-) create mode 100644 processes/cdksync/src/main/java/cz/inovatika/cdk/sync/utils/SyncUtils.java diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/kubernetes/KubernetesEnvSupport.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/kubernetes/KubernetesEnvSupport.java index 8ffbfc62b4..17b045e59f 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/kubernetes/KubernetesEnvSupport.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/utils/kubernetes/KubernetesEnvSupport.java @@ -21,7 +21,9 @@ public class KubernetesEnvSupport { public static final String TMSP_PREFIX = "TIMESTAMP_"; public static final String PROXY_API_PREFIX = "PROXY_API_"; public static final String REHARVEST_PREFIX = "REHARVEST_"; - public static final String ITERATION_SYNC_PREFIX = "ITERATION_SYNC_"; + + public static final String COMPARING_PREFIX = "COMPARING_"; + public static void prefixVariables(Map env, Map templateMap, String prefix) { env.keySet().forEach(key -> { @@ -68,10 +70,10 @@ public static Map iterationMap(Map env) { return iteration; } - public static Map iterationSyncMap(Map env) { - Map iteration = new HashMap<>(); - prefixVariables(env, iteration, ITERATION_SYNC_PREFIX); - return iteration; + public static Map comparingMap(Map env) { + Map comparing = new HashMap<>(); + prefixVariables(env, comparing, COMPARING_PREFIX); + return comparing; } public static Map proxyMap(Map env) { diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index ba4ca96484..730ef7d29f 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -53,7 +53,7 @@ public class ReharvestUtils { - private static final String ITERATION_ROWS_STRING_VALUE = "300"; + public static final String ITERATION_ROWS_STRING_VALUE = "300"; public static final Logger LOGGER = Logger.getLogger(ReharvestUtils.class.getName()); private ReharvestUtils() {} diff --git a/processes/cdksync/build.gradle b/processes/cdksync/build.gradle index e2ecd86a26..2f6697e40d 100644 --- a/processes/cdksync/build.gradle +++ b/processes/cdksync/build.gradle @@ -5,6 +5,7 @@ apply plugin: 'java' dependencies { api project(':shared:common') api project(':processes:cdkprocesses') + api project(':processes:cdkreharvest') implementation project(':processes:solr-migration') diff --git a/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java b/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java index 4041acd27b..6677437ec5 100644 --- a/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java +++ b/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java @@ -1,25 +1,60 @@ package cz.inovatika.cdk.sync; +import java.io.IOException; +import java.net.URLEncoder; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.LinkedHashSet; +import java.util.List; import java.util.Map; import java.util.TimeZone; +import java.util.TreeSet; +import java.util.UUID; import java.util.logging.Logger; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; +import javax.xml.parsers.ParserConfigurationException; + +import org.apache.commons.lang3.tuple.Pair; +import org.w3c.dom.Element; +import org.xml.sax.SAXException; + +import com.google.common.base.Functions; +import com.google.common.collect.Lists; import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; import com.sun.jersey.api.client.config.ClientConfig; import com.sun.jersey.api.client.config.DefaultClientConfig; import com.sun.jersey.api.json.JSONConfiguration; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem.TypeOfReharvset; +import cz.incad.kramerius.services.iterators.IterationItem; +import cz.incad.kramerius.services.iterators.ProcessIterator; +import cz.incad.kramerius.services.iterators.solr.SolrCursorIterator; +import cz.incad.kramerius.services.iterators.solr.SolrFilterQueryIterator; +import cz.incad.kramerius.services.iterators.solr.SolrPageIterator; +import cz.incad.kramerius.services.iterators.solr.SolrIteratorFactory.TypeOfIteration; +import cz.incad.kramerius.services.utils.SolrUtils; import cz.incad.kramerius.services.utils.kubernetes.KubernetesEnvSupport; +import cz.incad.kramerius.utils.StringUtils; +import cz.incad.kramerius.utils.conf.KConfiguration; +import static cz.incad.kramerius.utils.ReharvestUtils.*; -public class KubernetesSyncProcess { +public class KubernetesSyncProcess { + + + public static final Logger LOGGER = Logger.getLogger(KubernetesSyncProcess.class.getName()); public static final String ONLY_SHOW_CONFIGURATION = "ONLY_SHOW_CONFIGURATION"; - - protected static Client buildClient() { // Client client = Client.create(); ClientConfig cc = new DefaultClientConfig(); @@ -28,16 +63,194 @@ protected static Client buildClient() { return Client.create(cc); } - public static void main(String[] args) { + /** Find all pids by given root.pid */ + public static void comparePids(Map iterationMap,Map comparingMap,Map reharvestMap, String dl, String model, Client client) { - TimeZone.setDefault(TimeZone.getTimeZone("Europe/Prague")); + String reharvestUrl = reharvestMap.get("url"); + + HashSet source = identifiers(iterationMap, model, client); + TreeSet sortedSource = new TreeSet<>(source); + + HashSet comparing = identifiers(comparingMap, model, client); + TreeSet sortedComparing = new TreeSet<>(comparing); + + LOGGER.info(String.format("--- Model %s ---", model)); + + while(!sortedSource.isEmpty()) { + String sourceTop = sortedSource.first(); + if (comparing.contains(sourceTop)) { + sortedComparing.remove(sourceTop); + } else { + + // je ve zdroji(cdk) a neni v knihovne(knav, kfbz, nkp,... atd) => Potencionalne smazane dilo + // reharvest dle toho co je v indexu + + //https://api.val.ceskadigitalniknihovna.cz/search/api/client/v7.0 + String url = iterationMap.get("url"); + String endpoint = iterationMap.containsKey("endpoint") ? iterationMap.get("endpoint") : "select"; + String replaced = sourceTop.replace(":", "\\:"); + String query = URLEncoder.encode(String.format("pid:%s", replaced), "UTF-8")+"&wt=xml"; + + if (!url.endsWith("/")) { url = url+"/"; } + url = url + endpoint; + + Element executeQuery = SolrUtils.executeQuery(client, url, query, "", ""); + + + + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Sync trigger - reharvest from sync program","open", sourceTop, sourceTop); + reharvestItem.setTypeOfReharvest(TypeOfReharvset.root); + reharvestItem.setState("waiting_for_approve"); + reharvestItem.setRootPid(sourceTop); + reharvestItem.setOwnPidPath(sourceTop); + + + WebResource r = client.resource(reharvestUrl); + ClientResponse resp = r.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).entity(reharvestItem.toJSON().toString(), MediaType.APPLICATION_JSON).put(ClientResponse.class); + LOGGER.info("Status:"+resp.getStatus()); + if (resp.getStatus() != Response.Status.OK.getStatusCode()) { + String errorMsg = resp.getEntity(String.class); + LOGGER.warning(String.format("%s",errorMsg)); + } + } + sortedSource.remove(sourceTop); + } + + + while(!sortedComparing.isEmpty()) { + String compTop = sortedComparing.first(); + if (source.contains(compTop)) { + sortedSource.remove(compTop); + } else { + + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Sync trigger - reharvest from sync program","open", compTop, "none"); + reharvestItem.setTypeOfReharvest(TypeOfReharvset.new_root); + reharvestItem.setState("waiting_for_approve"); + reharvestItem.setRootPid(compTop); + reharvestItem.setLibraries(Arrays.asList(dl)); + + + WebResource r = client.resource(reharvestUrl); + ClientResponse resp = r.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).entity(reharvestItem.toJSON().toString(), MediaType.APPLICATION_JSON).put(ClientResponse.class); + LOGGER.info("Status:"+resp.getStatus()); + if (resp.getStatus() != Response.Status.OK.getStatusCode()) { + String errorMsg = resp.getEntity(String.class); + LOGGER.warning(String.format("%s",errorMsg)); + } + } + sortedComparing.remove(compTop); + } + } + + private static HashSet identifiers(Map map, String model, Client client) { + String iterationUrl = map.get("url"); + String api = map.get("api"); + + String masterQuery = "*:*"; + String filterQuery = ""; + if (map.containsKey("fq")) { + filterQuery = String.format("(%s AND %s)", map.get("fq"), modelQuery(model, api)); + } else { + filterQuery = modelQuery(model, api); + } + String sRows = map.containsKey("rows") ? map.get("rows") : ITERATION_ROWS_STRING_VALUE ; + String identifier = identifier(map); + TypeOfIteration typeOfIteration = typeOfIteration(map); + + + ProcessIterator processIterator = null; + switch (typeOfIteration) { + case CURSOR: { + processIterator = new SolrCursorIterator(iterationUrl, masterQuery, filterQuery, solrEndpoint(map), identifier(map), sort(map),Integer.parseInt(sRows)); + break; + } + case FILTER: { + processIterator = new SolrFilterQueryIterator( iterationUrl, masterQuery, filterQuery, solrEndpoint(map), identifier(map), sort(map),Integer.parseInt(sRows)); + break; + } + case PAGINATION: { + processIterator = new SolrPageIterator( iterationUrl, masterQuery, filterQuery, solrEndpoint(map), identifier(map), sort(map),Integer.parseInt(sRows)); + break; + } + } + LOGGER.info(String.format("Solr iterator %s, filter query %s, source url %s" , processIterator.getClass().getName(), filterQuery, iterationUrl)); + final HashSet sourcePids = new HashSet<>(); + processIterator.iterate(client, (list)-> { + for (IterationItem it : list) { + String id = it.getPid(); + if (identifier.equals("compositeId")) { + String[] arr = id.split("!"); + if (arr.length == 2) { + String pid = arr[1]; + sourcePids.add(pid); + } else { + sourcePids.add(id); + } + } else { + sourcePids.add(id); + } + } + + }, ()->{}); + + LOGGER.info(String.format("Number of results %d" , sourcePids.size())); + + return sourcePids; + } + + private static TypeOfIteration typeOfIteration(Map iterationMap) { + return iterationMap.containsKey("type") ? TypeOfIteration.valueOf(iterationMap.get("type")) : TypeOfIteration.CURSOR; + } + + private static String identifier(Map iterationMap) { + return iterationMap.containsKey("identifier") ? iterationMap.get("identifier") : "compositeId"; + } + + private static String sort(Map iterationMap) { + return iterationMap.containsKey("sort") ? iterationMap.get("sort") : String.format("%s asc", identifier(iterationMap)); + } + + private static String solrEndpoint(Map iterationMap) { + return iterationMap.containsKey("endpoint") ? iterationMap.get("endpoint") : "select"; + } + + private static String modelQuery(String model, String api) { + switch(api) { + case "v5": + return String.format("fedora.model:%s", model); + case "v7": + return String.format("model:%s", model); + } + return String.format("model:%s", model); + } + + public static void main(String[] args) { + TimeZone.setDefault(TimeZone.getTimeZone("Europe/Prague")); + + Client buildClient = buildClient(); Map env = System.getenv(); Map iterationMap = KubernetesEnvSupport.iterationMap(env); if (!iterationMap.containsKey("batch")) { iterationMap.put("batch", "45"); } + if (!iterationMap.containsKey("dl")) { + throw new RuntimeException("DL expecting"); + } + + if (!iterationMap.containsKey("fq")) { + iterationMap.put("fq", String.format("cdk.collection:%s", iterationMap.get("dl"))); + } + Map reharvestMap = KubernetesEnvSupport.reharvestMap(env); + Map comparingSyncMap = KubernetesEnvSupport.comparingMap(env); + if (!comparingSyncMap.containsKey("batch")) { + comparingSyncMap.put("batch", "45"); + } + List topLevelModels = Lists.transform(KConfiguration.getInstance().getConfiguration().getList("fedora.topLevelModels"), Functions.toStringFunction()); + for (String topLevelModel : topLevelModels) { + comparePids(iterationMap, comparingSyncMap,reharvestMap, iterationMap.get("dl"), topLevelModel, buildClient); + } } } diff --git a/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/utils/SyncUtils.java b/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/utils/SyncUtils.java new file mode 100644 index 0000000000..c85cc2fd7d --- /dev/null +++ b/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/utils/SyncUtils.java @@ -0,0 +1,7 @@ +package cz.inovatika.cdk.sync.utils; + +public class SyncUtils { + + private SyncUtils() {} + +} diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java index 6a52867202..5a9dbc0f1e 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java @@ -272,8 +272,27 @@ private void channelHealth(String library, JSONObject channelObject, JSONObject } private void usersDetail(JSONObject notLogged, JSONObject userJson) { - JSONArray rolesJSONArray = userJson.optJSONArray("roles"); - notLogged.put("roles", rolesJSONArray); + //JSONArray rolesDestArray = new JSONArray(); + JSONArray rolesSourceArray = userJson.optJSONArray("roles"); + JSONArray nRolesSourceArray = new JSONArray(); + if (rolesSourceArray != null) { + for (int i = 0; i < rolesSourceArray.length(); i++) { + Object obj = rolesSourceArray.get(i); + if (obj instanceof String) { + nRolesSourceArray.put(obj.toString()); + } else { + JSONObject roleObj = (JSONObject) obj; + String rname = roleObj.optString("name"); + if (rname != null) { + nRolesSourceArray.put(rname); + } + } + + } + } + + notLogged.put("roles", nRolesSourceArray); + if (userJson.has("labels")) { JSONArray licensesArray = userJson.getJSONArray("labels"); notLogged.put("licenses", licensesArray); diff --git a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java index ab88b6b988..c302e818a6 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java @@ -73,8 +73,9 @@ public static JSONObject checkUserChannelEndpoint(Client client, String ac,Strin WebResource configResource = client.resource(fullChannelUrl); Builder builder = null; if (header) { - builder = configResource.accept(MediaType.APPLICATION_JSON).header("CDK_TOKEN_PARAMETERS", "header_shib-session-id=_dd68cbd66641c9b647b05509ac0241fa|header_shib-session-expires=1592847906|header_shib-identity-provider=https://shibboleth.mzk.cz/simplesaml/metadata.xml|header_shib-authentication-method=urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport|header_shib-handler=https://dnnt.mzk.cz/Shibboleth.sso|header_eppn=test|header_entitlement=cdk.entitlement|header_eduPersonEntitlement=urn:mace:dir:entitlement:common-lib-terms|header_knav_type=validuser|header_knav_session_eppn=test|header_displayName=Inovatika|header_expiration_time=1720851072|header_knav_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_remote_user=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_eduPersonScopedAffiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_token_id=e96c9aec-a262-4fc6-8fe1-4104bdaa8dc8|header_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_knav_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_eduPersonPrincipalName=principalname@lib.cas.cz|header_knav_dnnt_user=test|header_eduPersonUniqueId=eduperson@lib.cas.cz|header_expires_in=1801|header_preffered_user_name=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_email=xxx@time.com|header_authentication_time=1720849271|header_ip_address=xx.xx.xx.xx"); - LOGGER.info("CDK_TOKEN_PARAMETERS = "+configResource+";"); + String headerText = "header_shib-session-id=_dd68cbd66641c9b647b05509ac0241fa|header_shib-session-expires=1592847906|header_shib-identity-provider=https://shibboleth.mzk.cz/simplesaml/metadata.xml|header_shib-authentication-method=urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport|header_shib-handler=https://dnnt.mzk.cz/Shibboleth.sso|header_eppn=test|header_entitlement=cdk.entitlement|header_eduPersonEntitlement=urn:mace:dir:entitlement:common-lib-terms|header_knav_type=validuser|header_knav_session_eppn=test|header_displayName=Inovatika|header_expiration_time=1720851072|header_knav_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_remote_user=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_eduPersonScopedAffiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_token_id=e96c9aec-a262-4fc6-8fe1-4104bdaa8dc8|header_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_knav_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_eduPersonPrincipalName=principalname@lib.cas.cz|header_knav_dnnt_user=test|header_eduPersonUniqueId=eduperson@lib.cas.cz|header_expires_in=1801|header_preffered_user_name=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_email=xxx@time.com|header_authentication_time=1720849271|header_ip_address=xx.xx.xx.xx"; + builder = configResource.accept(MediaType.APPLICATION_JSON).header("CDK_TOKEN_PARAMETERS", headerText); + LOGGER.info("CDK_TOKEN_PARAMETERS = "+headerText+";"); } else { builder = configResource.accept(MediaType.APPLICATION_JSON); } From 69f4e43d11217aa94c3b8183ede1390568fb65ac Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 21 Jul 2024 08:36:26 +0200 Subject: [PATCH 187/235] Fix dnnto users --- .../admin/v10/proxy/ConnectedInfoResource.java | 2 +- .../cz/incad/kramerius/cdk/ChannelUtils.java | 16 +++------------- 2 files changed, 4 insertions(+), 14 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java index 5a9dbc0f1e..9265155cea 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java @@ -257,7 +257,7 @@ private void channelHealth(String library, JSONObject channelObject, JSONObject JSONObject dnntJSON = ChannelUtils.checkUserChannelEndpoint(this.client, library, fullChannelUrl, true); usersDetail(dnntUser, dnntJSON); - usersObject.put("dnnt", notLoggedUser); + usersObject.put("dnnt", dnntJSON); } catch (Exception e) { diff --git a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java index c302e818a6..8fd2511812 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java @@ -64,20 +64,16 @@ public static void userChannelEndpoints(Client client, Map co LOGGER.info(String.format("Checking %s", fullChannelUrl)); checkUserChannelEndpoint(client, ac, fullChannelUrl,false); - - } } public static JSONObject checkUserChannelEndpoint(Client client, String ac,String fullChannelUrl, boolean header) { WebResource configResource = client.resource(fullChannelUrl); - Builder builder = null; + Builder builder = configResource.accept(MediaType.APPLICATION_JSON); if (header) { String headerText = "header_shib-session-id=_dd68cbd66641c9b647b05509ac0241fa|header_shib-session-expires=1592847906|header_shib-identity-provider=https://shibboleth.mzk.cz/simplesaml/metadata.xml|header_shib-authentication-method=urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport|header_shib-handler=https://dnnt.mzk.cz/Shibboleth.sso|header_eppn=test|header_entitlement=cdk.entitlement|header_eduPersonEntitlement=urn:mace:dir:entitlement:common-lib-terms|header_knav_type=validuser|header_knav_session_eppn=test|header_displayName=Inovatika|header_expiration_time=1720851072|header_knav_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_entitlement=urn:mace:dir:entitlement:common-lib-terms|header_remote_user=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_eduPersonScopedAffiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_token_id=e96c9aec-a262-4fc6-8fe1-4104bdaa8dc8|header_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_knav_affiliation=[employee@lib.cas.cz, member@lib.cas.cz]|header_eduPersonPrincipalName=principalname@lib.cas.cz|header_knav_dnnt_user=test|header_eduPersonUniqueId=eduperson@lib.cas.cz|header_expires_in=1801|header_preffered_user_name=f4ca5f6c5859d882f16aea477cd64a4c5887d3df824b91c7ab29c66091fccfff.aadzzwnyzxqx1rezk8w/sgpucjhb9hbrkxz8las3xof3hlpgnr6/ocwgyi82t6vwjepzgkru4iayuqkirk8dfilp68/i9ffozxdb25+wbrr8ij10tbowcfqgooztwhoiezaug/qijsyq1iftbo9cm5zq4z+h2ivqutjhv9trbjsdtnx0svpgtim=|header_email=xxx@time.com|header_authentication_time=1720849271|header_ip_address=xx.xx.xx.xx"; - builder = configResource.accept(MediaType.APPLICATION_JSON).header("CDK_TOKEN_PARAMETERS", headerText); + builder = builder.header("CDK_TOKEN_PARAMETERS", headerText); LOGGER.info("CDK_TOKEN_PARAMETERS = "+headerText+";"); - } else { - builder = configResource.accept(MediaType.APPLICATION_JSON); } ClientResponse userRes = builder.get(ClientResponse.class); if (userRes.getStatus() == ClientResponse.Status.OK.getStatusCode()) { @@ -100,13 +96,6 @@ public static void checkSolrChannelEndpoints(Client client, Map Date: Mon, 22 Jul 2024 10:15:10 +0200 Subject: [PATCH 188/235] Users endpoint fix --- .../v10/proxy/ConnectedInfoResource.java | 88 +++++++++++-------- .../redirection/item/V5ForwardHandler.java | 2 +- .../user/V5ForwardUserHandler.java | 3 - 3 files changed, 50 insertions(+), 43 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java index 9265155cea..c850f2e6ba 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java @@ -20,6 +20,7 @@ import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.commons.lang3.tuple.Pair; import org.apache.solr.client.solrj.SolrServerException; import org.json.JSONArray; import org.json.JSONObject; @@ -28,6 +29,7 @@ import com.sun.jersey.api.client.Client; import cz.incad.kramerius.cdk.ChannelUtils; +import cz.incad.kramerius.rest.api.k5.client.utils.UsersUtils; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance.InstanceType; @@ -220,8 +222,6 @@ public Response getChannelHealth(@PathParam("library") String library) { } private void channelHealth(String library, JSONObject channelObject, JSONObject usersObject) { - String baseurl = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".baseurl"); - String api = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".api","v5"); boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + library + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + library + ".licenses") : false; String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".forwardurl"); @@ -233,7 +233,7 @@ private void channelHealth(String library, JSONObject channelObject, JSONObject channelObject.put("enabled", true); // solr try { - String solrChannelUrl = ChannelUtils.solrChannelUrl(api, channel); + String solrChannelUrl = ChannelUtils.solrChannelUrl(inst.getInstanceType().name(), channel); ChannelUtils.checkSolrChannelEndpoint(this.client, library, solrChannelUrl); channelObject.put("solr", true); } catch (Exception e) { @@ -244,20 +244,19 @@ private void channelHealth(String library, JSONObject channelObject, JSONObject // user try { - String fullChannelUrl = ChannelUtils.userChannelUrl(api, channel); + String fullChannelUrl = ChannelUtils.userChannelUrl(inst.getInstanceType().name(), channel); - JSONObject notLoggedUser = new JSONObject(); - JSONObject dnntUser = new JSONObject(); - JSONObject notLoggedJSON = ChannelUtils.checkUserChannelEndpoint(this.client, library, fullChannelUrl, false); + Pair> notLogged = parsedUsers(inst, notLoggedJSON); + channelObject.put("user", true); - usersDetail(notLoggedUser, notLoggedJSON); - usersObject.put("notLogged", notLoggedUser); + usersObject.put("notLogged", UsersUtils.userToJSON(notLogged.getLeft(),notLogged.getRight(),false)); JSONObject dnntJSON = ChannelUtils.checkUserChannelEndpoint(this.client, library, fullChannelUrl, true); - usersDetail(dnntUser, dnntJSON); - usersObject.put("dnnt", dnntJSON); + Pair> dnntUser = parsedUsers(inst, dnntJSON); + + usersObject.put("dnnt", UsersUtils.userToJSON(dnntUser.getLeft(),dnntUser.getRight(),false)); } catch (Exception e) { @@ -271,37 +270,48 @@ private void channelHealth(String library, JSONObject channelObject, JSONObject } } - private void usersDetail(JSONObject notLogged, JSONObject userJson) { - //JSONArray rolesDestArray = new JSONArray(); - JSONArray rolesSourceArray = userJson.optJSONArray("roles"); - JSONArray nRolesSourceArray = new JSONArray(); - if (rolesSourceArray != null) { - for (int i = 0; i < rolesSourceArray.length(); i++) { - Object obj = rolesSourceArray.get(i); - if (obj instanceof String) { - nRolesSourceArray.put(obj.toString()); - } else { - JSONObject roleObj = (JSONObject) obj; - String rname = roleObj.optString("name"); - if (rname != null) { - nRolesSourceArray.put(rname); - } - } - - } - } - - notLogged.put("roles", nRolesSourceArray); - - if (userJson.has("labels")) { - JSONArray licensesArray = userJson.getJSONArray("labels"); - notLogged.put("licenses", licensesArray); - } else if (userJson.has("licenses")){ - JSONArray licensesArray = userJson.getJSONArray("licenses"); - notLogged.put("licenses", licensesArray); + private Pair> parsedUsers(OneInstance inst, JSONObject notLoggedJSON) { + switch(inst.getInstanceType()) { + case V5: + return V5ForwardUserHandler.userFromJSON(notLoggedJSON); + case V7: + return V7ForwardUserHandler.userFromJSON(notLoggedJSON); + default: + return V7ForwardUserHandler.userFromJSON(notLoggedJSON); } } +// private void usersDetail(JSONObject notLogged, JSONObject userJson) { +// //JSONArray rolesDestArray = new JSONArray(); +// JSONArray rolesSourceArray = userJson.optJSONArray("roles"); +// JSONArray nRolesSourceArray = new JSONArray(); +// if (rolesSourceArray != null) { +// for (int i = 0; i < rolesSourceArray.length(); i++) { +// Object obj = rolesSourceArray.get(i); +// if (obj instanceof String) { +// nRolesSourceArray.put(obj.toString()); +// } else { +// JSONObject roleObj = (JSONObject) obj; +// String rname = roleObj.optString("name"); +// if (rname != null) { +// nRolesSourceArray.put(rname); +// } +// } +// +// } +// } +// +// notLogged.put("roles", nRolesSourceArray); +// +// if (userJson.has("labels")) { +// JSONArray licensesArray = userJson.getJSONArray("labels"); +// notLogged.put("licenses", licensesArray); +// } else if (userJson.has("licenses")){ +// JSONArray licensesArray = userJson.getJSONArray("licenses"); +// notLogged.put("licenses", licensesArray); +// } +// } + @PUT @Path("{library}/timestamp") diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java index 1fce92af29..5b8fee4a45 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java @@ -23,7 +23,7 @@ import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.x; import cz.incad.kramerius.rest.apiNew.client.v60.filter.ProxyFilter; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java index 53d07b793d..224096470b 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java @@ -41,13 +41,10 @@ protected String forwardUrl() { @Override public Pair> user() throws ProxyHandlerException { String baseurl = forwardUrl(); - // https://kramerius-dnnt.lib.cas.cz/search/api/v5.0/cdk/forward/user String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/user"; ClientResponse fResponse = super.forwardedResponse(url); String entity = fResponse.getEntity(String.class); JSONObject jObject = new JSONObject(entity); - // LOGGER.info("Collecting user information from = "+url+"; information = - // "+jObject.toString()); return userFromJSON(jObject); } From ac6e4dd57afadde4f1819c51ab6c35ee985a6e04 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 22 Jul 2024 10:20:58 +0200 Subject: [PATCH 189/235] Reharvest manager fix --- .../apiNew/client/v60/redirection/item/V5ForwardHandler.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java index 5b8fee4a45..1fce92af29 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java @@ -23,7 +23,7 @@ import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.x; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.filter.ProxyFilter; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; From b5881d07b83809c12f5fffc8a9ae909ef5c57682 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 22 Jul 2024 11:38:03 +0200 Subject: [PATCH 190/235] Users utils fix --- .../user/V5ForwardUserHandler.java | 76 +++++++++--------- .../user/V7ForwardUserHandler.java | 78 +++++++++---------- 2 files changed, 73 insertions(+), 81 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java index 224096470b..4387054198 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java @@ -49,50 +49,44 @@ public Pair> user() throws ProxyHandlerException { } public static Pair> userFromJSON(JSONObject json) throws JSONException { - if (json.has("id") && json.getInt("id") > -1) { - - Map session = new HashMap<>(); - String loginName = json.optString("lname"); - String firstName = json.optString("firstname"); - String surName = json.optString("surname"); - ; - - List licenses = new ArrayList<>(); - List roles = new ArrayList<>(); - JSONArray rolesJSONArray = json.optJSONArray("roles"); - - if (rolesJSONArray != null) { - for (int i = 0; i < rolesJSONArray.length(); i++) { - String optString = rolesJSONArray.getJSONObject(i).optString("name"); - roles.add(new RoleImpl(optString)); - } - } - - if (json.has("session")) { - JSONObject sessionAttrs = json.getJSONObject("session"); - sessionAttrs.keySet().forEach(key -> { - Object object = sessionAttrs.get((String) key); - session.put(key.toString(), object.toString()); - }); + Map session = new HashMap<>(); + String loginName = json.optString("lname"); + String firstName = json.optString("firstname"); + String surName = json.optString("surname"); + ; + + List licenses = new ArrayList<>(); + List roles = new ArrayList<>(); + JSONArray rolesJSONArray = json.optJSONArray("roles"); + + if (rolesJSONArray != null) { + for (int i = 0; i < rolesJSONArray.length(); i++) { + String optString = rolesJSONArray.getJSONObject(i).optString("name"); + roles.add(new RoleImpl(optString)); } + } - if (json.has("labels")) { - JSONArray licensesArray = json.getJSONArray("labels"); - for (int i = 0; i < licensesArray.length(); i++) { - licenses.add(licensesArray.getString(i)); - } - } - - UserImpl userImpl = new UserImpl(1, firstName != null ? firstName : "", surName != null ? surName : "", - loginName, 0); - userImpl.setGroups(roles.toArray(new Role[roles.size()])); - session.entrySet().forEach(entry -> { - userImpl.addSessionAttribute(entry.getKey(), entry.getValue()); + if (json.has("session")) { + JSONObject sessionAttrs = json.getJSONObject("session"); + sessionAttrs.keySet().forEach(key -> { + Object object = sessionAttrs.get((String) key); + session.put(key.toString(), object.toString()); }); + } - return Pair.of(userImpl, licenses); - - } else - return null; + if (json.has("labels")) { + JSONArray licensesArray = json.getJSONArray("labels"); + for (int i = 0; i < licensesArray.length(); i++) { + licenses.add(licensesArray.getString(i)); + } + } + + UserImpl userImpl = new UserImpl(1, firstName != null ? firstName : "", surName != null ? surName : "", + loginName, 0); + userImpl.setGroups(roles.toArray(new Role[roles.size()])); + session.entrySet().forEach(entry -> { + userImpl.addSessionAttribute(entry.getKey(), entry.getValue()); + }); + return Pair.of(userImpl, licenses); } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java index 655adfe1d1..1780f76727 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java @@ -54,52 +54,50 @@ public Pair> user() throws ProxyHandlerException { } public static Pair> userFromJSON(JSONObject json) throws JSONException { - if (json.optBoolean(AUTHENTICATED_KEY, false)) { - String uid = json.getString(UID_KEY); - String name = json.optString(NAME_KEY,""); - String[] names = name.split("\\s"); - String firstName = null; - String secondName = null; - if (names.length > 0) firstName = names[0]; - if (names.length > 1) secondName = names[1]; - - Map session = new HashMap<>(); - List licenses = new ArrayList<>(); - List roles = new ArrayList<>(); - JSONArray rolesJSONArray = json.optJSONArray(ROLES_KEY); - - if (rolesJSONArray != null) { - for (int i = 0; i < rolesJSONArray.length(); i++) { - String optString = rolesJSONArray.getString(i); - roles.add(new RoleImpl(optString)); - } - } - if (json.has(SESSION_KEY)) { - JSONObject sessionAttrs = json.getJSONObject(SESSION_KEY); - sessionAttrs.keySet().forEach(key -> { - Object object = sessionAttrs.get((String) key); - session.put(key.toString(), object.toString()); - }); + + String uid = json.getString(UID_KEY); + String name = json.optString(NAME_KEY,""); + String[] names = name.split("\\s"); + String firstName = null; + String secondName = null; + if (names.length > 0) firstName = names[0]; + if (names.length > 1) secondName = names[1]; + + Map session = new HashMap<>(); + List licenses = new ArrayList<>(); + List roles = new ArrayList<>(); + JSONArray rolesJSONArray = json.optJSONArray(ROLES_KEY); + + if (rolesJSONArray != null) { + for (int i = 0; i < rolesJSONArray.length(); i++) { + String optString = rolesJSONArray.getString(i); + roles.add(new RoleImpl(optString)); } + } - if (json.has(LICENSES_KEY)) { - JSONArray licensesArray = json.getJSONArray(LICENSES_KEY); - for (int i = 0; i < licensesArray.length(); i++) { - licenses.add(licensesArray.getString(i)); - } - } + if (json.has(SESSION_KEY)) { + JSONObject sessionAttrs = json.getJSONObject(SESSION_KEY); + sessionAttrs.keySet().forEach(key -> { + Object object = sessionAttrs.get((String) key); + session.put(key.toString(), object.toString()); + }); + } - UserImpl userImpl = new UserImpl(1, firstName != null ? firstName : "", secondName != null ? secondName : "", uid, 0); - userImpl.setGroups(roles.toArray(new Role[roles.size()])); + if (json.has(LICENSES_KEY)) { + JSONArray licensesArray = json.getJSONArray(LICENSES_KEY); + for (int i = 0; i < licensesArray.length(); i++) { + licenses.add(licensesArray.getString(i)); + } + } - session.entrySet().forEach(entry -> { - userImpl.addSessionAttribute(entry.getKey(), entry.getValue()); - }); + UserImpl userImpl = new UserImpl(1, firstName != null ? firstName : "", secondName != null ? secondName : "", uid, 0); + userImpl.setGroups(roles.toArray(new Role[roles.size()])); - return Pair.of(userImpl, licenses); + session.entrySet().forEach(entry -> { + userImpl.addSessionAttribute(entry.getKey(), entry.getValue()); + }); - } else - return null; + return Pair.of(userImpl, licenses); } } From 6b43f9bfbe60afb7759e87b252fb221775360d03 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 22 Jul 2024 13:44:09 +0200 Subject: [PATCH 191/235] Docker image/Kubernetes support --- processes/cdksync/build.gradle | 7 +++--- .../cdk/sync/KubernetesSyncProcess.java | 23 ++++++++----------- 2 files changed, 13 insertions(+), 17 deletions(-) diff --git a/processes/cdksync/build.gradle b/processes/cdksync/build.gradle index 2f6697e40d..9548f70073 100644 --- a/processes/cdksync/build.gradle +++ b/processes/cdksync/build.gradle @@ -1,4 +1,4 @@ -//apply plugin: 'application' +apply plugin: 'application' apply plugin: 'java' @@ -30,7 +30,6 @@ configurations.api { } -/* application { - mainClass = 'cz.incad.kramerius.KubernetesReharvestProcess' -}*/ + mainClass = 'cz.inovatika.cdk.sync.KubernetesSyncProcess' +} diff --git a/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java b/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java index 6677437ec5..dddb5971dc 100644 --- a/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java +++ b/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java @@ -1,6 +1,7 @@ package cz.inovatika.cdk.sync; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.net.URLEncoder; import java.util.ArrayList; import java.util.Arrays; @@ -63,8 +64,11 @@ protected static Client buildClient() { return Client.create(cc); } - /** Find all pids by given root.pid */ - public static void comparePids(Map iterationMap,Map comparingMap,Map reharvestMap, String dl, String model, Client client) { + /** Find all pids by given root.pid + * @throws IOException + * @throws SAXException + * @throws ParserConfigurationException */ + public static void comparePids(Map iterationMap,Map comparingMap,Map reharvestMap, String dl, String model, Client client) throws ParserConfigurationException, SAXException, IOException { String reharvestUrl = reharvestMap.get("url"); @@ -82,32 +86,25 @@ public static void comparePids(Map iterationMap,Map Potencionalne smazane dilo - // reharvest dle toho co je v indexu - //https://api.val.ceskadigitalniknihovna.cz/search/api/client/v7.0 String url = iterationMap.get("url"); String endpoint = iterationMap.containsKey("endpoint") ? iterationMap.get("endpoint") : "select"; - String replaced = sourceTop.replace(":", "\\:"); - String query = URLEncoder.encode(String.format("pid:%s", replaced), "UTF-8")+"&wt=xml"; + //String replaced = sourceTop.replace(":", "\\:"); + //String query = URLEncoder.encode(String.format("pid:%s", replaced), "UTF-8")+"&wt=xml"; if (!url.endsWith("/")) { url = url+"/"; } url = url + endpoint; - Element executeQuery = SolrUtils.executeQuery(client, url, query, "", ""); - - ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Sync trigger - reharvest from sync program","open", sourceTop, sourceTop); reharvestItem.setTypeOfReharvest(TypeOfReharvset.root); reharvestItem.setState("waiting_for_approve"); reharvestItem.setRootPid(sourceTop); reharvestItem.setOwnPidPath(sourceTop); - WebResource r = client.resource(reharvestUrl); ClientResponse resp = r.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).entity(reharvestItem.toJSON().toString(), MediaType.APPLICATION_JSON).put(ClientResponse.class); - LOGGER.info("Status:"+resp.getStatus()); + //LOGGER.info("Status:"+resp.getStatus()); if (resp.getStatus() != Response.Status.OK.getStatusCode()) { String errorMsg = resp.getEntity(String.class); LOGGER.warning(String.format("%s",errorMsg)); @@ -225,7 +222,7 @@ private static String modelQuery(String model, String api) { } - public static void main(String[] args) { + public static void main(String[] args) throws ParserConfigurationException, SAXException, IOException { TimeZone.setDefault(TimeZone.getTimeZone("Europe/Prague")); Client buildClient = buildClient(); From d0451d3ae10583f775029036b9a05fdb73188879 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 22 Jul 2024 16:45:42 +0200 Subject: [PATCH 192/235] Rehavetst - conflict response --- .../admin/v10/reharvest/ReharvestManager.java | 2 + .../v10/reharvest/ReharvestResource.java | 9 ++-- .../impl/MemoryReharvestManagerImpl.java | 9 +++- .../impl/SolrReharvestManagerImpl.java | 46 +++++++++++++++++++ 4 files changed, 61 insertions(+), 5 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java index 0835d07ba0..13ec4d06ef 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestManager.java @@ -17,6 +17,8 @@ public interface ReharvestManager { public ReharvestItem getTopItem(String status); public ReharvestItem getItemById(String id); + + public ReharvestItem getOpenItemByPid(String pid); public void deregister(String id); } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index fe116143b4..39a349d167 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -155,11 +155,13 @@ public boolean acceptElement(Element element) { // nasel to v indexu (pro polozky, ktere nejsou v indexu je potreba novy typ harvestu ReharvestItem item = ReharvestItem.fromJSON(jsonObj); - - switch(item.getTypeOfReharvest()) { + ReharvestItem alreadyRegistredItem = this.reharvestManager.getOpenItemByPid(item.getPid()); + if (alreadyRegistredItem != null) { + return Response.status(Response.Status.CONFLICT).build(); + } else { + switch(item.getTypeOfReharvest()) { case root: case children: - if (item != null && StringUtils.isAnyString(item.getRootPid()) && StringUtils.isAnyString(item.getOwnPidPath())) { item.setTimestamp(Instant.now()); this.reharvestManager.register(item); @@ -182,6 +184,7 @@ public boolean acceptElement(Element element) { } default: throw new IllegalStateException(String.format("Uknown type of reharvest %s", item.getTypeOfReharvest())); + } } } catch (JSONException | ParseException e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java index 1cc4820735..ca5fc92210 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/MemoryReharvestManagerImpl.java @@ -60,8 +60,13 @@ public ReharvestItem getTopItem(String status) { public ReharvestItem getItemById(String id) { return this.mapper.get(id); } - - + + @Override + public ReharvestItem getOpenItemByPid(String pid) { + // TODO Auto-generated method stub + return null; + } + @Override public ReharvestItem update(ReharvestItem item) throws UnsupportedEncodingException, JSONException, ParseException { return item; diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java index 2e1b6fd8e4..aa4eba3c6f 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java @@ -177,6 +177,52 @@ public ReharvestItem getItemById(String id) { return null; } + + + + + @Override + public ReharvestItem getOpenItemByPid(String pid) { + try { + String reharvest = KConfiguration.getInstance().getSolrReharvestHost(); + String query = String.format("pid:(%s)+AND+state:(open+OR+waiting_for_approve)", pid); + String sort = URLEncoder.encode("indexed asc","UTF-8"); + String fullUrl = String.format("%s/select?q=%s&rows=1&sort=%s", reharvest, query,sort); + String t = solrGet(fullUrl); + JSONObject solrResp = new JSONObject(t); + JSONArray docs = solrDocs(solrResp); + if (docs.length() > 0) { + JSONObject doc = docs.getJSONObject(0); + ReharvestItem reharvestItem = ReharvestItem.fromJSON(doc); + return reharvestItem; + } + } catch (UniformInterfaceException | ClientHandlerException | JSONException | ParseException | UnsupportedEncodingException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + } + return null; + } + +// @Override +// public ReharvestItem getItemByPid(String pid) { +// try { +// String reharvest = KConfiguration.getInstance().getSolrReharvestHost(); +// String query = String.format("pid:(%s)", pid); +// String sort = URLEncoder.encode("indexed asc","UTF-8"); +// String fullUrl = String.format("%s/select?q=%s&rows=1&sort=%s", reharvest, query,sort); +// String t = solrGet(fullUrl); +// JSONObject solrResp = new JSONObject(t); +// JSONArray docs = solrDocs(solrResp); +// if (docs.length() > 0) { +// JSONObject doc = docs.getJSONObject(0); +// ReharvestItem reharvestItem = ReharvestItem.fromJSON(doc); +// return reharvestItem; +// } +// } catch (UniformInterfaceException | ClientHandlerException | JSONException | ParseException | UnsupportedEncodingException e) { +// LOGGER.log(Level.SEVERE, e.getMessage(), e); +// } +// return null; +// } + private JSONArray solrDocs(JSONObject solrResp) { JSONArray docs = new JSONArray(); if (solrResp.has("response")) { From 13e31b0f74cc02866e8fd42cae5ceb371d642e85 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 22 Jul 2024 17:00:31 +0200 Subject: [PATCH 193/235] Reharvest resource - new endpoint --- .../cdk/sync/KubernetesSyncProcess.java | 103 ++++++++++++++---- .../v10/reharvest/ReharvestResource.java | 18 ++- 2 files changed, 99 insertions(+), 22 deletions(-) diff --git a/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java b/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java index dddb5971dc..5451f6e318 100644 --- a/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java +++ b/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java @@ -10,6 +10,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TimeZone; import java.util.TreeSet; import java.util.UUID; @@ -20,6 +21,7 @@ import javax.xml.parsers.ParserConfigurationException; import org.apache.commons.lang3.tuple.Pair; +import org.json.JSONObject; import org.w3c.dom.Element; import org.xml.sax.SAXException; @@ -68,8 +70,8 @@ protected static Client buildClient() { * @throws IOException * @throws SAXException * @throws ParserConfigurationException */ - public static void comparePids(Map iterationMap,Map comparingMap,Map reharvestMap, String dl, String model, Client client) throws ParserConfigurationException, SAXException, IOException { - + public static void comparePids(Map iterationMap,Map comparingMap,Map reharvestMap, JSONObject libs, String dl, String model, Client client) throws ParserConfigurationException, SAXException, IOException { + String reharvestUrl = reharvestMap.get("url"); HashSet source = identifiers(iterationMap, model, client); @@ -86,7 +88,6 @@ public static void comparePids(Map iterationMap,Map iterationMap,Map iterationMap,Map allEnabledLibraries(JSONObject libs) { + List libsArray = new ArrayList<>(); + Set keys = libs.keySet(); + for (Object key : keys) { + JSONObject libObject = libs.getJSONObject(key.toString()); + boolean status = libObject.optBoolean("status",false); + if (status) libsArray.add(key.toString()); + } + return libsArray; + } + + private static boolean exists(String pid, Map map, Client client) throws ParserConfigurationException, SAXException, IOException { + String iterationUrl = map.get("url"); + + String solrEndpoint = solrEndpoint(map); + //String identifier = identifier(map); + + String masterQuery = URLEncoder.encode(String.format("%s:\"%s\"", "pid", pid), "UTF-8"); + + + Element response = SolrUtils.executeQuery(client, iterationUrl, solrEndpoint+"?q="+masterQuery+"&wt=xml", "", ""); + List findAllPids = SolrUtils.findAllPids(response); + + return !findAllPids.isEmpty(); + } + + private static HashSet identifiers(Map map, String model, Client client) { String iterationUrl = map.get("url"); String api = map.get("api"); @@ -244,10 +293,22 @@ public static void main(String[] args) throws ParserConfigurationException, SAXE if (!comparingSyncMap.containsKey("batch")) { comparingSyncMap.put("batch", "45"); } - - List topLevelModels = Lists.transform(KConfiguration.getInstance().getConfiguration().getList("fedora.topLevelModels"), Functions.toStringFunction()); - for (String topLevelModel : topLevelModels) { - comparePids(iterationMap, comparingSyncMap,reharvestMap, iterationMap.get("dl"), topLevelModel, buildClient); + + Map proxyMap = KubernetesEnvSupport.proxyMap(env); + String proxyUrl = proxyMap.get("url"); + if (proxyUrl == null) throw new IllegalStateException("expecting PROXY_API_URL"); + WebResource r = buildClient.resource(proxyUrl); + ClientResponse resp = r.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).get(ClientResponse.class); + if (resp.getStatus() == Response.Status.OK.getStatusCode()) { + String proxyURLREsp = resp.getEntity(String.class); + JSONObject libraries = new JSONObject(proxyURLREsp); + List topLevelModels = Lists.transform(KConfiguration.getInstance().getConfiguration().getList("fedora.topLevelModels"), Functions.toStringFunction()); + for (String topLevelModel : topLevelModels) { + comparePids(iterationMap, comparingSyncMap,reharvestMap, libraries, iterationMap.get("dl"), topLevelModel, buildClient); + } + } else { + throw new RuntimeException(String.format("Error response from %s (status %d)",proxyUrl, resp.getStatus())); } } + } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index 39a349d167..bb7627521d 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -211,7 +211,23 @@ public Response getTopHarvest(@QueryParam("state")String state) { } } - + + @GET + @Path("open") + @Produces(MediaType.APPLICATION_JSON) + public Response getOpenHarvest(@QueryParam("pid")String pid) { + if (StringUtils.isAnyString(pid)) { + ReharvestItem topItem = this.reharvestManager.getOpenItemByPid(pid); + if (topItem != null) { + return Response.ok(topItem.toJSON().toString()).build(); + } else { + return Response.status(Response.Status.NOT_FOUND).build(); + } + } else { + return Response.status(Response.Status.BAD_REQUEST).build(); + } + } + @PUT @Path("{id}/state") @Produces(MediaType.APPLICATION_JSON) From 22e7a47652001ad6020f50598934cdd49b19917c Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 22 Jul 2024 17:21:44 +0200 Subject: [PATCH 194/235] Debug message --- .../rest/apiNew/admin/v10/reharvest/ReharvestResource.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index bb7627521d..ab97b04bd0 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -216,14 +216,17 @@ public Response getTopHarvest(@QueryParam("state")String state) { @Path("open") @Produces(MediaType.APPLICATION_JSON) public Response getOpenHarvest(@QueryParam("pid")String pid) { + LOGGER.info("Returning open harvest"); if (StringUtils.isAnyString(pid)) { ReharvestItem topItem = this.reharvestManager.getOpenItemByPid(pid); if (topItem != null) { return Response.ok(topItem.toJSON().toString()).build(); } else { + LOGGER.info("Open harvest not found"); return Response.status(Response.Status.NOT_FOUND).build(); } } else { + LOGGER.info("Open harvest Bad request"); return Response.status(Response.Status.BAD_REQUEST).build(); } } From 9ca75850c70fa4bfe771511e2cf8e1c38cd6df2e Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 22 Jul 2024 17:38:09 +0200 Subject: [PATCH 195/235] Fixed query --- .../admin/v10/reharvest/impl/SolrReharvestManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java index aa4eba3c6f..d8f1c1651b 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java @@ -185,7 +185,7 @@ public ReharvestItem getItemById(String id) { public ReharvestItem getOpenItemByPid(String pid) { try { String reharvest = KConfiguration.getInstance().getSolrReharvestHost(); - String query = String.format("pid:(%s)+AND+state:(open+OR+waiting_for_approve)", pid); + String query = URLEncoder.encode( String.format("pid:(\"%s\")+AND+state:(open+OR+waiting_for_approve)", pid),"UTF-8"); String sort = URLEncoder.encode("indexed asc","UTF-8"); String fullUrl = String.format("%s/select?q=%s&rows=1&sort=%s", reharvest, query,sort); String t = solrGet(fullUrl); From 19d2f1a70e04863a9f072801900e7741a3bcc69a Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 22 Jul 2024 18:05:15 +0200 Subject: [PATCH 196/235] Fixed query --- .../admin/v10/reharvest/impl/SolrReharvestManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java index d8f1c1651b..e27e8483f1 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java @@ -185,7 +185,7 @@ public ReharvestItem getItemById(String id) { public ReharvestItem getOpenItemByPid(String pid) { try { String reharvest = KConfiguration.getInstance().getSolrReharvestHost(); - String query = URLEncoder.encode( String.format("pid:(\"%s\")+AND+state:(open+OR+waiting_for_approve)", pid),"UTF-8"); + String query = URLEncoder.encode( String.format("pid:(\"%s\") AND state:(open OR waiting_for_approve)", pid),"UTF-8"); String sort = URLEncoder.encode("indexed asc","UTF-8"); String fullUrl = String.format("%s/select?q=%s&rows=1&sort=%s", reharvest, query,sort); String t = solrGet(fullUrl); From 1c3cb819f80552ff099ccae262e2c9bf9e817d7b Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 23 Jul 2024 11:30:08 +0200 Subject: [PATCH 197/235] Delete trigger - avoid multiple registration --- .../cdk/sync/KubernetesSyncProcess.java | 73 ++++++++++++++++--- .../impl/SolrReharvestManagerImpl.java | 2 +- .../v60/redirection/ProxyHandlerSupport.java | 21 +++--- 3 files changed, 73 insertions(+), 23 deletions(-) diff --git a/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java b/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java index 5451f6e318..c0a947438e 100644 --- a/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java +++ b/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java @@ -77,6 +77,35 @@ public static void comparePids(Map iterationMap,Map source = identifiers(iterationMap, model, client); TreeSet sortedSource = new TreeSet<>(source); + if (!comparingMap.containsKey("api")) { + JSONObject libObject = libs.getJSONObject(dl); + JSONObject configObject = libObject.getJSONObject("config"); + comparingMap.put("api", configObject.getString("api")); + } + if (!comparingMap.containsKey("url")) { + JSONObject libObject = libs.getJSONObject(dl); + JSONObject configObject = libObject.getJSONObject("config"); + String forwardUrl = configObject.getString("forwardurl"); + String api = comparingMap.get("api"); + switch (api) { + case "v7": + String v7url= forwardUrl+(forwardUrl.endsWith("/")?"":"/")+ "api/cdk/v7.0/forward/sync/solr"; + comparingMap.put("url", v7url); + break; + + case "v5": + String v5url= forwardUrl+(forwardUrl.endsWith("/")?"":"/")+ "api/v5.0/cdk/forward/sync/solr"; + comparingMap.put("url", v5url); + break; + + default: + String defaulturl= forwardUrl+(forwardUrl.endsWith("/")?"":"/")+ "api/cdk/v7.0/forward/sync/solr"; + comparingMap.put("url", defaulturl); + break; + } + //comparingMap.put("url", configObject.getString("forwardurl")); + } + HashSet comparing = identifiers(comparingMap, model, client); TreeSet sortedComparing = new TreeSet<>(comparing); @@ -90,14 +119,12 @@ public static void comparePids(Map iterationMap,Map iterationMap,Map iterationMap,Map allEnabledLibraries(JSONObject libs) { for (Object key : keys) { JSONObject libObject = libs.getJSONObject(key.toString()); boolean status = libObject.optBoolean("status",false); - if (status) libsArray.add(key.toString()); + if (status) { + JSONObject config = libObject.getJSONObject("config"); + if (config.optBoolean("licenses",false)) { + libsArray.add(key.toString()); + } + } } return libsArray; } @@ -176,7 +208,6 @@ private static boolean exists(String pid, Map map, Client clien String iterationUrl = map.get("url"); String solrEndpoint = solrEndpoint(map); - //String identifier = identifier(map); String masterQuery = URLEncoder.encode(String.format("%s:\"%s\"", "pid", pid), "UTF-8"); @@ -293,19 +324,39 @@ public static void main(String[] args) throws ParserConfigurationException, SAXE if (!comparingSyncMap.containsKey("batch")) { comparingSyncMap.put("batch", "45"); } - + + Map proxyMap = KubernetesEnvSupport.proxyMap(env); String proxyUrl = proxyMap.get("url"); if (proxyUrl == null) throw new IllegalStateException("expecting PROXY_API_URL"); + + JSONObject librariesInfo = librariesInfo(buildClient, proxyUrl); + List topLevelModels = Lists.transform(KConfiguration.getInstance().getConfiguration().getList("fedora.topLevelModels"), Functions.toStringFunction()); + for (String topLevelModel : topLevelModels) { + comparePids(iterationMap, comparingSyncMap,reharvestMap, librariesInfo, iterationMap.get("dl"), topLevelModel, buildClient); + } + } + + + + private static JSONObject librariesInfo(Client buildClient, String proxyUrl) { WebResource r = buildClient.resource(proxyUrl); ClientResponse resp = r.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).get(ClientResponse.class); if (resp.getStatus() == Response.Status.OK.getStatusCode()) { String proxyURLREsp = resp.getEntity(String.class); JSONObject libraries = new JSONObject(proxyURLREsp); - List topLevelModels = Lists.transform(KConfiguration.getInstance().getConfiguration().getList("fedora.topLevelModels"), Functions.toStringFunction()); - for (String topLevelModel : topLevelModels) { - comparePids(iterationMap, comparingSyncMap,reharvestMap, libraries, iterationMap.get("dl"), topLevelModel, buildClient); + Set keys = libraries.keySet(); + for (Object key : keys) { + JSONObject libObject = libraries.getJSONObject(key.toString()); + WebResource keyr = buildClient.resource(proxyUrl+"/"+key.toString()+"/config"); + ClientResponse configResp = keyr.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).get(ClientResponse.class); + if (configResp.getStatus() == Response.Status.OK.getStatusCode()) { + String cionfigResp = configResp.getEntity(String.class); + JSONObject configRespJSON = new JSONObject(cionfigResp); + libObject.put("config", configRespJSON); + } } + return libraries; } else { throw new RuntimeException(String.format("Error response from %s (status %d)",proxyUrl, resp.getStatus())); } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java index e27e8483f1..9cd9313965 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java @@ -185,7 +185,7 @@ public ReharvestItem getItemById(String id) { public ReharvestItem getOpenItemByPid(String pid) { try { String reharvest = KConfiguration.getInstance().getSolrReharvestHost(); - String query = URLEncoder.encode( String.format("pid:(\"%s\") AND state:(open OR waiting_for_approve)", pid),"UTF-8"); + String query = URLEncoder.encode( String.format("pid:(\"%s\") AND NOT state:(closed OR cancelled)", pid),"UTF-8"); String sort = URLEncoder.encode("indexed asc","UTF-8"); String fullUrl = String.format("%s/select?q=%s&rows=1&sort=%s", reharvest, query,sort); String t = solrGet(fullUrl); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index f5a86580e8..cceefc8fe2 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -224,19 +224,18 @@ public boolean acceptElement(Element element) { pidPath = pidPath.substring(0, index + ownParentPidText.length()).trim(); } try { - ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", ownParentPid.getTextContent().trim(), pidPath); - reharvestItem.setTypeOfReharvest(TypeOfReharvset.children); - reharvestItem.setState("waiting_for_approve"); - - if (cdkCollection != null) { - List collections = XMLUtils.getElements(cdkCollection).stream().map(Element::getTextContent).collect(Collectors.toList()); - reharvestItem.setLibraries(collections); + ReharvestItem alreadyRegistredItem = this.reharvestManager.getOpenItemByPid(ownParentPid.getTextContent().trim()); + if (alreadyRegistredItem == null) { + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", ownParentPid.getTextContent().trim(), pidPath); + reharvestItem.setTypeOfReharvest(TypeOfReharvset.children); + reharvestItem.setState("waiting_for_approve"); + if (cdkCollection != null) { + List collections = XMLUtils.getElements(cdkCollection).stream().map(Element::getTextContent).collect(Collectors.toList()); + reharvestItem.setLibraries(collections); + } + this.reharvestManager.register(reharvestItem); } - - //LOGGER.info(String.format("Registering item %s", reharvestItem.toJSON().toString())); - - this.reharvestManager.register(reharvestItem); } catch (DOMException e) { LOGGER.log(Level.SEVERE,e.getMessage(),e); } catch (AlreadyRegistedPidsException e) { From f1a7df668437601c294244644fdd650471ef3b24 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 23 Jul 2024 12:20:11 +0200 Subject: [PATCH 198/235] failed state in filter --- .../admin/v10/reharvest/impl/SolrReharvestManagerImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java index 9cd9313965..872ebe6460 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/impl/SolrReharvestManagerImpl.java @@ -185,7 +185,7 @@ public ReharvestItem getItemById(String id) { public ReharvestItem getOpenItemByPid(String pid) { try { String reharvest = KConfiguration.getInstance().getSolrReharvestHost(); - String query = URLEncoder.encode( String.format("pid:(\"%s\") AND NOT state:(closed OR cancelled)", pid),"UTF-8"); + String query = URLEncoder.encode( String.format("pid:(\"%s\") AND NOT state:(closed OR cancelled OR failed)", pid),"UTF-8"); String sort = URLEncoder.encode("indexed asc","UTF-8"); String fullUrl = String.format("%s/select?q=%s&rows=1&sort=%s", reharvest, query,sort); String t = solrGet(fullUrl); From e9b7ca6f354f51414fdfaf89217da6f627ac92f8 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 24 Jul 2024 14:56:51 +0200 Subject: [PATCH 199/235] Reharvests & sync --- .../iterators/solr/SolrPageIterator.java | 11 +- .../kramerius/KubernetesReharvestProcess.java | 6 +- .../incad/kramerius/utils/ReharvestUtils.java | 12 +- .../cdk/sync/KubernetesSyncProcess.java | 165 +++++++++--------- .../admin/v10/reharvest/ReharvestItem.java | 7 +- .../v10/reharvest/ReharvestResource.java | 11 ++ .../v60/redirection/ProxyHandlerSupport.java | 24 ++- .../redirection/item/V5ForwardHandler.java | 18 +- .../redirection/item/V7ForwardHandler.java | 14 +- 9 files changed, 154 insertions(+), 114 deletions(-) diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/solr/SolrPageIterator.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/solr/SolrPageIterator.java index a9898656b5..7313c81575 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/solr/SolrPageIterator.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/iterators/solr/SolrPageIterator.java @@ -13,6 +13,7 @@ import javax.xml.parsers.ParserConfigurationException; import java.io.IOException; import java.net.URLEncoder; +import java.util.List; import static cz.incad.kramerius.services.iterators.utils.IterationUtils.pidsToIterationItem; import static cz.incad.kramerius.services.utils.SolrUtils.*; @@ -31,7 +32,7 @@ public SolrPageIterator(String address, String masterQuery, String filterQuery, super(address, masterQuery, filterQuery, endpoint, id, sorting, rows, user, pass); } - public static Element paginationQuery(Client client, String url, String mq, String offset, int rows, String filterQuery, String endpoint, String identifierField, String user, String pass) throws IOException, SAXException, ParserConfigurationException { + public static Element paginationQuery(Client client, String url, String mq, String offset, int rows, String filterQuery, String endpoint, String identifierField, String sorting, String user, String pass) throws IOException, SAXException, ParserConfigurationException { String fullQuery = null; if (StringUtils.isAnyString(filterQuery)) { fullQuery = String.format("?q=%s&start=%s&rows=%d&fq=%s&fl=%s",mq,offset, rows, URLEncoder.encode(filterQuery,"UTF-8"), identifierField); @@ -39,6 +40,9 @@ public static Element paginationQuery(Client client, String url, String mq, Stri fullQuery = String.format("?q=%s&start=%s&rows=%d&fl=%s",mq,offset, rows,identifierField); } String query = endpoint+ fullQuery+"&wt=xml"; + if (StringUtils.isAnyString(sorting)) { + query = query+"&sort="+URLEncoder.encode(sorting,"UTF-8"); + } return SolrUtils.executeQuery(client, url, query, user, pass); } @@ -64,11 +68,12 @@ public void iterate(Client client, ProcessIterationCallback iterationCallback, P int offset = 0; int numberOfResult = Integer.MAX_VALUE; do { - Element element = paginationQuery( client, address,masterQuery, ""+offset, rows, filterQuery, endpoint, id, this.user, this.pass); + Element element = paginationQuery( client, address,masterQuery, ""+offset, rows, filterQuery, endpoint, id, this.sorting, this.user, this.pass); if (numberOfResult == Integer.MAX_VALUE) { numberOfResult = findNumberOfResults(element); } - iterationCallback.call(pidsToIterationItem(this.address, findAllPids(element))); + List allPids = findAllPids(element); + iterationCallback.call(pidsToIterationItem(this.address, allPids)); offset += rows; }while(offset < numberOfResult); // callback after iteration diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java index 09756b8e24..68b4f87361 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -35,6 +35,7 @@ import antlr.StringUtils; import cz.incad.kramerius.cdk.ChannelUtils; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; +import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem.TypeOfReharvset; import cz.incad.kramerius.service.MigrateSolrIndexException; import cz.incad.kramerius.services.utils.kubernetes.KubernetesEnvSupport; import cz.incad.kramerius.utils.ReharvestUtils; @@ -113,8 +114,11 @@ public static void main(String[] args) { ChannelUtils.checkSolrChannelEndpoints(client, configurations); // delete all asociated pids from index ReharvestUtils.deleteAllGivenPids(client, destinationMap, allPidsList, onlyShowConfiguration); + // reindex pids - ReharvestUtils.reharvestPIDFromGivenCollections(pid, configurations, ""+onlyShowConfiguration, destinationMap, iterationMap, reharvestItem); + if (!reharvestItem.getTypeOfReharvest().equals(TypeOfReharvset.delete_pid) && !reharvestItem.getTypeOfReharvest().equals(TypeOfReharvset.delete_tree)) { + ReharvestUtils.reharvestPIDFromGivenCollections(pid, configurations, ""+onlyShowConfiguration, destinationMap, iterationMap, reharvestItem); + } if (!onlyShowConfiguration) { changeState(client, wurl, id,"closed"); diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index 730ef7d29f..91525b109d 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -101,6 +101,9 @@ public static List> findPidByType(Map iterat String masterQuery = "*:*"; String filterQuery = "none"; switch(item.getTypeOfReharvest()) { + case only_pid: + filterQuery = "pid:\"" + item.getPid() + "\""; + break; case new_root: case root: filterQuery = "root.pid:\"" + item.getRootPid() + "\""; @@ -178,7 +181,14 @@ public static String renderTemplate(String api, Map iteration, Ma public static String fq(String api, String pid, ReharvestItem item) throws UnsupportedEncodingException { switch(item.getTypeOfReharvest()) { - + + case only_pid: + switch(api) { + case "v7": return "pid:\""+pid+"\""; + case "v5": return "PID:\""+pid+"\""; + } + return "PID:\""+pid+"\""; + case new_root: case root: switch(api) { diff --git a/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java b/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java index c0a947438e..8fd9a61aae 100644 --- a/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java +++ b/processes/cdksync/src/main/java/cz/inovatika/cdk/sync/KubernetesSyncProcess.java @@ -6,6 +6,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.LinkedHashSet; import java.util.List; @@ -75,7 +76,7 @@ public static void comparePids(Map iterationMap,Map source = identifiers(iterationMap, model, client); - TreeSet sortedSource = new TreeSet<>(source); + //TreeSet sortedSource = new TreeSet<>(source); if (!comparingMap.containsKey("api")) { JSONObject libObject = libs.getJSONObject(dl); @@ -103,88 +104,77 @@ public static void comparePids(Map iterationMap,Map comparing = identifiers(comparingMap, model, client); - TreeSet sortedComparing = new TreeSet<>(comparing); - + LOGGER.info(String.format("--- Model %s ---", model)); - - while(!sortedSource.isEmpty()) { - String sourceTop = sortedSource.first(); - if (comparing.contains(sourceTop)) { - sortedComparing.remove(sourceTop); - } else { - - String url = iterationMap.get("url"); - String endpoint = iterationMap.containsKey("endpoint") ? iterationMap.get("endpoint") : "select"; - - if (!url.endsWith("/")) { url = url+"/"; } - url = url + endpoint; - - - ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Sync trigger - probably deleted- whole reharvest","open", sourceTop, sourceTop); - reharvestItem.setTypeOfReharvest(TypeOfReharvset.root); - reharvestItem.setState("waiting_for_approve"); - reharvestItem.setRootPid(sourceTop); - reharvestItem.setOwnPidPath(sourceTop); - - reharvestItem.setLibraries(allEnabledLibraries(libs)); + List probablyDeleted = new ArrayList<>(source); - - WebResource r = client.resource(reharvestUrl); - ClientResponse resp = r.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).entity(reharvestItem.toJSON().toString(), MediaType.APPLICATION_JSON).put(ClientResponse.class); - if (resp.getStatus() != Response.Status.OK.getStatusCode()) { - String errorMsg = resp.getEntity(String.class); - LOGGER.warning(String.format("%s",errorMsg)); - } - } - sortedSource.remove(sourceTop); + probablyDeleted.removeAll(comparing); + LOGGER.info(String.format("Probably deleted titles %s",probablyDeleted)); + + for (String pid : probablyDeleted) { + + String url = iterationMap.get("url"); + String endpoint = iterationMap.containsKey("endpoint") ? iterationMap.get("endpoint") : "select"; + + if (!url.endsWith("/")) { url = url+"/"; } + url = url + endpoint; + + + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Sync trigger|probably deleted","open", pid, pid); + reharvestItem.setTypeOfReharvest(TypeOfReharvset.root); + reharvestItem.setState("waiting_for_approve"); + reharvestItem.setRootPid(pid); + reharvestItem.setOwnPidPath(pid); + + reharvestItem.setLibraries(allEnabledLibraries(libs)); + + WebResource r = client.resource(reharvestUrl); + ClientResponse resp = r.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).entity(reharvestItem.toJSON().toString(), MediaType.APPLICATION_JSON).put(ClientResponse.class); + if (resp.getStatus() != Response.Status.OK.getStatusCode()) { + String errorMsg = resp.getEntity(String.class); + LOGGER.warning(String.format("%s",errorMsg)); + } } - - while(!sortedComparing.isEmpty()) { - String compTop = sortedComparing.first(); - if (source.contains(compTop)) { - sortedSource.remove(compTop); - } else { - - if (exists(compTop, iterationMap, client)) { - - ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Sync trigger - missing DL","open", compTop, "none"); - reharvestItem.setTypeOfReharvest(TypeOfReharvset.root); - reharvestItem.setState("waiting_for_approve"); - reharvestItem.setRootPid(compTop); - reharvestItem.setLibraries(allEnabledLibraries(libs)); - - - WebResource r = client.resource(reharvestUrl); - ClientResponse resp = r.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).entity(reharvestItem.toJSON().toString(), MediaType.APPLICATION_JSON).put(ClientResponse.class); - LOGGER.info("Status:"+resp.getStatus()); - if (resp.getStatus() != Response.Status.OK.getStatusCode()) { - String errorMsg = resp.getEntity(String.class); - LOGGER.warning(String.format("%s",errorMsg)); - } - - } else { - ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Sync trigger - missing title","open", compTop, "none"); - reharvestItem.setTypeOfReharvest(TypeOfReharvset.new_root); - reharvestItem.setState("waiting_for_approve"); - reharvestItem.setRootPid(compTop); - reharvestItem.setLibraries(Arrays.asList(dl)); - - - WebResource r = client.resource(reharvestUrl); - ClientResponse resp = r.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).entity(reharvestItem.toJSON().toString(), MediaType.APPLICATION_JSON).put(ClientResponse.class); - LOGGER.info("Status:"+resp.getStatus()); - if (resp.getStatus() != Response.Status.OK.getStatusCode()) { - String errorMsg = resp.getEntity(String.class); - LOGGER.warning(String.format("%s",errorMsg)); - } - } - } - sortedComparing.remove(compTop); + + List probablyNotHarvested = new ArrayList<>(comparing); + probablyNotHarvested.removeAll(source); + LOGGER.info(String.format("Probably not harvested titles %s",probablyNotHarvested)); + + for (String pid : probablyNotHarvested) { + if (exists(pid, iterationMap, client)) { + // muze byt i spatny model??? - pak konflikt + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Sync trigger|missing DL","open", pid, "none"); + reharvestItem.setTypeOfReharvest(TypeOfReharvset.root); + reharvestItem.setState("waiting_for_approve"); + reharvestItem.setRootPid(pid); + reharvestItem.setLibraries(allEnabledLibraries(libs)); + + WebResource r = client.resource(reharvestUrl); + ClientResponse resp = r.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).entity(reharvestItem.toJSON().toString(), MediaType.APPLICATION_JSON).put(ClientResponse.class); + LOGGER.info("Status:"+resp.getStatus()); + if (resp.getStatus() != Response.Status.OK.getStatusCode()) { + String errorMsg = resp.getEntity(String.class); + LOGGER.warning(String.format("%s",errorMsg)); + } + } else { + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Sync trigger|missing title","open", pid, "none"); + reharvestItem.setTypeOfReharvest(TypeOfReharvset.new_root); + reharvestItem.setState("waiting_for_approve"); + reharvestItem.setRootPid(pid); + reharvestItem.setLibraries(Arrays.asList(dl)); + + WebResource r = client.resource(reharvestUrl); + ClientResponse resp = r.accept(MediaType.APPLICATION_JSON).type(MediaType.APPLICATION_JSON).entity(reharvestItem.toJSON().toString(), MediaType.APPLICATION_JSON).put(ClientResponse.class); + LOGGER.info("Status:"+resp.getStatus()); + if (resp.getStatus() != Response.Status.OK.getStatusCode()) { + String errorMsg = resp.getEntity(String.class); + LOGGER.warning(String.format("%s",errorMsg)); + } + } } } @@ -206,15 +196,10 @@ private static List allEnabledLibraries(JSONObject libs) { private static boolean exists(String pid, Map map, Client client) throws ParserConfigurationException, SAXException, IOException { String iterationUrl = map.get("url"); - String solrEndpoint = solrEndpoint(map); - String masterQuery = URLEncoder.encode(String.format("%s:\"%s\"", "pid", pid), "UTF-8"); - - Element response = SolrUtils.executeQuery(client, iterationUrl, solrEndpoint+"?q="+masterQuery+"&wt=xml", "", ""); List findAllPids = SolrUtils.findAllPids(response); - return !findAllPids.isEmpty(); } @@ -252,8 +237,12 @@ private static HashSet identifiers(Map map, String model } LOGGER.info(String.format("Solr iterator %s, filter query %s, source url %s" , processIterator.getClass().getName(), filterQuery, iterationUrl)); final HashSet sourcePids = new HashSet<>(); + //final HashMap> pp = new HashMap<>(); + //List compositeIds = new ArrayList<>(); processIterator.iterate(client, (list)-> { + LOGGER.info("SourcePids size "+sourcePids.size()); for (IterationItem it : list) { + String id = it.getPid(); if (identifier.equals("compositeId")) { String[] arr = id.split("!"); @@ -267,7 +256,7 @@ private static HashSet identifiers(Map map, String model sourcePids.add(id); } } - + }, ()->{}); LOGGER.info(String.format("Number of results %d" , sourcePids.size())); @@ -331,8 +320,16 @@ public static void main(String[] args) throws ParserConfigurationException, SAXE if (proxyUrl == null) throw new IllegalStateException("expecting PROXY_API_URL"); JSONObject librariesInfo = librariesInfo(buildClient, proxyUrl); - List topLevelModels = Lists.transform(KConfiguration.getInstance().getConfiguration().getList("fedora.topLevelModels"), Functions.toStringFunction()); - for (String topLevelModel : topLevelModels) { + + List iterationModels = new ArrayList<>(); + if (!iterationMap.containsKey("models")) { + iterationModels = Lists.transform(KConfiguration.getInstance().getConfiguration().getList("fedora.topLevelModels"), Functions.toStringFunction()); + } else { + String[] models = iterationMap.get("models").split(","); + iterationModels = Arrays.asList(models); + } + + for (String topLevelModel : iterationModels) { comparePids(iterationMap, comparingSyncMap,reharvestMap, librariesInfo, iterationMap.get("dl"), topLevelModel, buildClient); } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java index 1a48fc857d..fb8614ab5e 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestItem.java @@ -26,7 +26,12 @@ public static enum TypeOfReharvset { children, // reharvest titulu a potomku new_children, // dilo v cdk smazano a je potreba stahnout z krameriu informace o detech a titulu a pak pomoci own_pid_path - new_root; // dilo je v cdk smazano a je potreba stahnot z krameriu informace o korenovem dile + new_root, // dilo je v cdk smazano a je potreba stahnot z krameriu informace o korenovem dile + + only_pid, // pouze jeden pid + + delete_pid, + delete_tree; } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index ab97b04bd0..41191b160e 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -160,6 +160,17 @@ public boolean acceptElement(Element element) { return Response.status(Response.Status.CONFLICT).build(); } else { switch(item.getTypeOfReharvest()) { + + case only_pid: + if (item != null && StringUtils.isAnyString(item.getPid())) { + item.setTimestamp(Instant.now()); + this.reharvestManager.register(item); + return Response.ok(item.toJSON().toString()).build(); + } else { + JSONObject errorObject = new JSONObject(); + errorObject.put("error", "No pid"); + return Response.status(Response.Status.BAD_REQUEST).entity(errorObject.toString()).type(MediaType.APPLICATION_JSON_TYPE).build(); + } case root: case children: if (item != null && StringUtils.isAnyString(item.getRootPid()) && StringUtils.isAnyString(item.getOwnPidPath())) { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index cceefc8fe2..1c668172b8 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -44,6 +44,7 @@ import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem.TypeOfReharvset; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.StringUtils; import cz.incad.kramerius.utils.XMLUtils; @@ -109,16 +110,16 @@ public ClientResponse forwardedResponse(String url) throws ProxyHandlerException } } - public Response buildForwardResponseGET(String url) throws ProxyHandlerException { - return buildForwardResponseGET(url, null, null); + public Response buildForwardResponseGET(String url, boolean deleteTrigger) throws ProxyHandlerException { + return buildForwardResponseGET(url, null, null, deleteTrigger); } - public Response buildForwardResponseGET(String url, String pid) throws ProxyHandlerException { - return buildForwardResponseGET(url, null, pid); + public Response buildForwardResponseGET(String url, String pid, boolean deleteTrigger) throws ProxyHandlerException { + return buildForwardResponseGET(url, null, pid, deleteTrigger); } - public Response buildForwardResponseGET(String url, String mimetype, String pid) throws ProxyHandlerException { + public Response buildForwardResponseGET(String url, String mimetype, String pid, boolean deleteTrigger) throws ProxyHandlerException { WebResource.Builder b = buidFowrardResponse(url); ClientResponse response = b.get(ClientResponse.class); if (response.getStatus() == 200) { @@ -157,7 +158,7 @@ public void write(OutputStream output) throws IOException, WebApplicationExcepti } else { // event for reharvest if (response.getStatus() == 404) { - deleteTriggeToReharvest(pid); + if (deleteTrigger) deleteTriggeToReharvest(pid); } return Response.status(response.getStatus()).build(); } @@ -226,11 +227,18 @@ public boolean acceptElement(Element element) { try { ReharvestItem alreadyRegistredItem = this.reharvestManager.getOpenItemByPid(ownParentPid.getTextContent().trim()); if (alreadyRegistredItem == null) { - ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger - reharvest from core","open", ownParentPid.getTextContent().trim(), pidPath); + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger|404 ","open", ownParentPid.getTextContent().trim(), pidPath); reharvestItem.setTypeOfReharvest(TypeOfReharvset.children); reharvestItem.setState("waiting_for_approve"); if (cdkCollection != null) { - List collections = XMLUtils.getElements(cdkCollection).stream().map(Element::getTextContent).collect(Collectors.toList()); + // all libraries + List collections = new ArrayList<>(); + List enabledInstances = this.instances.enabledInstances(); + for (OneInstance inst : enabledInstances) { + String acronym = inst.getName(); + boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + acronym + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + acronym + ".licenses") : false; + if (channelAccess) { collections.add(acronym); } + } reharvestItem.setLibraries(collections); } this.reharvestManager.register(reharvestItem); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java index 1fce92af29..61714dd32f 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java @@ -57,7 +57,7 @@ public Response image(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, null); + return buildForwardResponseGET(url, null, true); } } @@ -70,7 +70,7 @@ public Response imagePreview(RequestMethodName method) throws ProxyHandlerExcept if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, null); + return buildForwardResponseGET(url, null,false); } } @@ -87,7 +87,7 @@ public Response mods(RequestMethodName method) throws ProxyHandlerException { + "/streams/BIBLIO_MODS"; // String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/item/" // + this.pid + "/streams/BIBLIO_MODS"; - return buildForwardResponseGET(url, this.pid); + return buildForwardResponseGET(url, this.pid, true); } } @@ -100,7 +100,7 @@ public Response zoomifyImageProperties(RequestMethodName method) throws ProxyHan if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, null); + return buildForwardResponseGET(url, null, true); } } @@ -109,7 +109,7 @@ public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHan String baseurl = this.forwardUrl(); String formatted = String.format("api/v5.0/cdk/forward/zoomify/%s/%s/%s", this.pid, tileGroupStr, tileStr); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + formatted; - return buildForwardResponseGET(url, null); + return buildForwardResponseGET(url, null, false); } @Override @@ -121,7 +121,7 @@ public Response textOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url,null); + return buildForwardResponseGET(url,null, false); } } @@ -134,7 +134,7 @@ public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, null); + return buildForwardResponseGET(url, null, false); } } @@ -169,7 +169,7 @@ public Response iiifInfo(RequestMethodName method, String pid) throws ProxyHandl if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, null); + return buildForwardResponseGET(url, null, true); } } @@ -192,7 +192,7 @@ public Response iiifTile(RequestMethodName method, String pid, String region, St if (splited.length > 1) { mime = IIIF_SUPPORTED_MIMETYPES.containsKey(splited[1]) ? IIIF_SUPPORTED_MIMETYPES.get(splited[1]) : defaultMime; } - return buildForwardResponseGET(url, null, mime); + return buildForwardResponseGET(url, null, mime,false); } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index e81b4bf13c..75235a0fc9 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -99,7 +99,7 @@ public Response image(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, null); + return buildForwardResponseGET(url, null, true); } } @@ -112,7 +112,7 @@ public Response imagePreview(RequestMethodName method) throws ProxyHandlerExcept if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, null); + return buildForwardResponseGET(url, null,false); } } @@ -124,7 +124,7 @@ public Response mods(RequestMethodName method) throws ProxyHandlerException { String baseurl = this.forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/item/" + this.pid + "/streams/BIBLIO_MODS"; - return buildForwardResponseGET(url,this.pid); + return buildForwardResponseGET(url,this.pid, true); } } @@ -137,7 +137,7 @@ public Response zoomifyImageProperties(RequestMethodName method) throws ProxyHan String baseurl = forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/zoomify/" + this.pid + "/ImageProperties.xml"; - return buildForwardResponseGET(url, null); + return buildForwardResponseGET(url, null, true); } } @@ -148,7 +148,7 @@ public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHan String baseurl = forwardUrl(); String formatted = String.format("api/cdk/v7.0/forward/zoomify/%s/%s/%s", this.pid, tileGroupStr, tileStr); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + formatted; - return buildForwardResponseGET(url, null); + return buildForwardResponseGET(url, null, false); } @@ -161,7 +161,7 @@ public Response textOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, null); + return buildForwardResponseGET(url, null, false); } } @@ -173,7 +173,7 @@ public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, null); + return buildForwardResponseGET(url, null,false); } } From 66343f78aa51ca51d88b64d8ef580e0f9c23b71c Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 24 Jul 2024 15:43:31 +0200 Subject: [PATCH 200/235] Reharvests type --- .../src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index 91525b109d..12df92eade 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -102,12 +102,14 @@ public static List> findPidByType(Map iterat String filterQuery = "none"; switch(item.getTypeOfReharvest()) { case only_pid: + case delete_pid: filterQuery = "pid:\"" + item.getPid() + "\""; break; case new_root: case root: filterQuery = "root.pid:\"" + item.getRootPid() + "\""; break; + case delete_tree: case children: String ownPidPath = item.getOwnPidPath(); if (!StringUtils.isAnyString(ownPidPath)) { From 892f57d569e2dc2dd2732ae7fe0f6f11bc22161d Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 24 Jul 2024 15:44:27 +0200 Subject: [PATCH 201/235] New type of reharvests --- .../rest/apiNew/admin/v10/reharvest/ReharvestResource.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java index 41191b160e..f8b0b37cce 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/reharvest/ReharvestResource.java @@ -161,6 +161,7 @@ public boolean acceptElement(Element element) { } else { switch(item.getTypeOfReharvest()) { + case delete_pid: case only_pid: if (item != null && StringUtils.isAnyString(item.getPid())) { item.setTimestamp(Instant.now()); @@ -171,6 +172,7 @@ public boolean acceptElement(Element element) { errorObject.put("error", "No pid"); return Response.status(Response.Status.BAD_REQUEST).entity(errorObject.toString()).type(MediaType.APPLICATION_JSON_TYPE).build(); } + case delete_tree: case root: case children: if (item != null && StringUtils.isAnyString(item.getRootPid()) && StringUtils.isAnyString(item.getOwnPidPath())) { From 38274a0aba3613b3ada71ffd1004d5780343915d Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 15 Aug 2024 10:46:19 +0200 Subject: [PATCH 202/235] hl.fragsize & hl.snippets limits --- .../kramerius/KubernetesReharvestProcess.java | 22 +++- .../incad/kramerius/utils/ReharvestUtils.java | 109 ++++++++++++++++++ .../apiNew/client/v60/SearchResource.java | 55 ++++++++- 3 files changed, 181 insertions(+), 5 deletions(-) diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java index 68b4f87361..e90e837a70 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/KubernetesReharvestProcess.java @@ -117,9 +117,29 @@ public static void main(String[] args) { // reindex pids if (!reharvestItem.getTypeOfReharvest().equals(TypeOfReharvset.delete_pid) && !reharvestItem.getTypeOfReharvest().equals(TypeOfReharvset.delete_tree)) { + if (reharvestItem.getTypeOfReharvest().equals(TypeOfReharvset.new_children)) { + Map> foundRoots = ReharvestUtils.findRootsAndPidPathsFromLibs(client,pid,configurations); + LOGGER.info("Found roots:"+foundRoots); + //String foundPid = null; + String foundRootPid = null; + String ownPidPath = null; + for (String key : foundRoots.keySet()) { + foundRootPid = foundRoots.get(key).get("root.pid"); + ownPidPath = foundRoots.get(key).get("own_pid_path"); + } + if (ownPidPath != null) { + LOGGER.info("Own pid path "+ownPidPath); + reharvestItem.setOwnPidPath(ownPidPath); + } + if (foundRootPid != null) { + LOGGER.info("root pid "+ownPidPath); + reharvestItem.setRootPid(foundRootPid); + } + } + + ReharvestUtils.reharvestPIDFromGivenCollections(pid, configurations, ""+onlyShowConfiguration, destinationMap, iterationMap, reharvestItem); } - if (!onlyShowConfiguration) { changeState(client, wurl, id,"closed"); } diff --git a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java index 12df92eade..9521940bf0 100644 --- a/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java +++ b/processes/cdkreharvest/src/main/java/cz/incad/kramerius/utils/ReharvestUtils.java @@ -9,6 +9,7 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStream; +import java.io.StringReader; import java.io.StringWriter; import java.io.UnsupportedEncodingException; import java.net.URLEncoder; @@ -17,10 +18,12 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; +import javax.ws.rs.core.MediaType; import javax.xml.parsers.ParserConfigurationException; import javax.xml.transform.TransformerException; @@ -30,12 +33,15 @@ import org.apache.commons.lang3.tuple.Pair; import org.json.JSONArray; import org.json.JSONObject; +import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.xml.sax.SAXException; import com.google.common.io.Files; import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.KubernetesReharvestProcess; import cz.incad.kramerius.TooBigException; @@ -219,7 +225,110 @@ public static String fq(String api, String pid, ReharvestItem item) throws Unsup } return "root_pid:\""+pid+"\""; } + + public static Map> findRootsAndPidPathsFromLibs(Client client, String pid, Map collectionConfigurations){ + Map> retval = new HashMap<>(); + for (String ac : collectionConfigurations.keySet()) { + JSONObject colObject = collectionConfigurations.get(ac); + + String apiVersion = colObject.optString("api","v5"); + if (!colObject.has("forwardurl")) { + LOGGER.severe(String.format("Skipping %s", ac)); + continue; + } + //Map iteration = new HashMap<>(iterationMap); + String channel = colObject.optString("forwardurl"); + + String fullChannelUrl = ChannelUtils.solrChannelUrl(apiVersion, channel); + String pidIdentifier = "pid"; + String rootPid ="root.pid"; + String pidPath = "own_pid_path"; + switch (apiVersion) { + case "v5": + pidIdentifier="PID"; + rootPid="root_pid"; + pidPath = "pid_path"; + break; + case "v7": + pidIdentifier="pid"; + rootPid="root.pid"; + pidPath = "own_pid_path"; + break; + default: + pidIdentifier = "pid"; + rootPid="root.pid"; + pidPath = "pid_path"; + break; + } + + String query = String.format("%s:\"%s\"", pidIdentifier,pid);; + WebResource solrResource = client.resource(fullChannelUrl+ String.format("/select?q=%s&rows=1&wt=xml",query)); + ClientResponse solrREsp = solrResource.accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + if (solrREsp.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + String str = solrREsp.getEntity(String.class); + try { + + final String finalPidIdentifier = pidIdentifier; + final String finalRootPid = rootPid; + final String finalPidPath = rootPid; + + + Document parsed = XMLUtils.parseDocument(new StringReader(str)); + Map map = new HashMap<>(); + + Element pidElement = XMLUtils.findElement(parsed.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String fieldName = element.getAttribute("name"); + return fieldName.equals(finalPidIdentifier); + } + }); + if (pidElement != null) { + map.put("pid", pidElement.getTextContent()); + } + + Element pidPathElement = XMLUtils.findElement(parsed.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String fieldName = element.getAttribute("name"); + return fieldName.equals(finalRootPid); + } + }); + if (pidPathElement != null) { + List elms = XMLUtils.getElements(pidPathElement); + if (!elms.isEmpty()) { + List pids = elms.stream().map(Element::getTextContent).collect(Collectors.toList()); + if (pids.size() >0 ) { + map.put("own_pid_path", pids.get(0)); + } + } else { + map.put("own_pid_path", pidPathElement.getTextContent()); + } + } + + Element rootPidElm = XMLUtils.findElement(parsed.getDocumentElement(), new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String fieldName = element.getAttribute("name"); + return fieldName.equals(finalPidPath); + } + }); + if (rootPidElm != null) { + map.put("root.pid", rootPidElm.getTextContent()); + } + + retval.put(ac, map); + + } catch (DOMException | ParserConfigurationException | SAXException | IOException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } + } + } + return retval; + } + public static void reharvestPIDFromGivenCollections(String pid, Map collectionConfigurations, String onlyShowConfiguration, Map destinationMap, Map iterationMap, ReharvestItem item) throws IllegalAccessException, InstantiationException, ClassNotFoundException, NoSuchMethodException, MigrateSolrIndexException, IOException, ParserConfigurationException, SAXException { List harvestFiles = new ArrayList<>(); for (String ac : collectionConfigurations.keySet()) { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java index c639752fd1..3525eb6c07 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/SearchResource.java @@ -24,6 +24,7 @@ import cz.incad.kramerius.rest.apiNew.exceptions.BadRequestException; import cz.incad.kramerius.rest.apiNew.exceptions.InternalErrorException; import cz.incad.kramerius.security.User; +import cz.incad.kramerius.solr.SolrKeys; import cz.incad.kramerius.utils.XMLUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.lang.StringUtils; @@ -166,7 +167,11 @@ private String buildSearchResponseXml(UriInfo uriInfo) { } private String buildSearchSolrQueryString(UriInfo uriInfo) throws UnsupportedEncodingException { - boolean fqFound = false; + + int snippets = -1; + int fragsize = -1; + + boolean fqFound = false; MultivaluedMap queryParameters = uriInfo.getQueryParameters(); StringBuilder builder = new StringBuilder(); @@ -181,9 +186,16 @@ private String buildSearchSolrQueryString(UriInfo uriInfo) throws UnsupportedEnc if (k.equals("fq")) { value = this.proxyFilter.enhancedFilter(value); } + if (k.equals("hl.fragsize")) { - value = normalizeHighlightFragsize(value).toString(); + value = validateHighlightFragsize(value).toString(); + fragsize = Integer.parseInt( value ); } + if (k.equals("hl.snippets")) { + value = validateHighlightSnippets(value).toString(); + snippets = Integer.parseInt( value ); + } + builder.append(k).append("=").append(URLEncoder.encode(value, "UTF-8")); builder.append("&"); } @@ -219,15 +231,50 @@ private void checkFlValueDoesNotContainFilteredField(String comaSeparatedValues) } } - private Integer normalizeHighlightFragsize(String value) { +// private Integer normalizeHighlightFragsize(String value) { +// try { +// Integer hlFragSize = Integer.valueOf(value); +// return hlFragSize > MAX_FRAG_SIZE ? MAX_FRAG_SIZE : hlFragSize; +// } catch (NumberFormatException e) { +// throw new BadRequestException(e.getMessage()); +// } +// } + + + private Integer validateHighlightFragsize(String value) { try { Integer hlFragSize = Integer.valueOf(value); - return hlFragSize > MAX_FRAG_SIZE ? MAX_FRAG_SIZE : hlFragSize; + if (hlFragSize > SolrKeys.MAX_HL_FRAGSIZE) { + throw new BadRequestException(String.format("The value of the parameter hl.fragsize is too large (%d). The maximum allowed value is %d.", hlFragSize, SolrKeys.MAX_HL_FRAGSIZE)); + } + if (hlFragSize == 0) { + throw new BadRequestException("The value of the parameter hl.fragsize cannot be 0"); + } + return hlFragSize ; } catch (NumberFormatException e) { throw new BadRequestException(e.getMessage()); } } + private Integer validateHighlightSnippets(String value) { + try { + Integer hlSnippets = Integer.valueOf(value); + if (hlSnippets > SolrKeys.MAX_HL_SNIPPETS) { + throw new BadRequestException(String.format("The value of the parameter hl.snippet is too large (%d). The maximum allowed value is %d.", hlSnippets, SolrKeys.MAX_HL_SNIPPETS)); + } + return hlSnippets; + } catch (NumberFormatException e) { + throw new BadRequestException(e.getMessage()); + } + } + + + private void validateHLCombination(int fragsize, int snippets) { + int combination = fragsize * snippets; + if (combination > SolrKeys.MAX_HL_COMBINATION) { + throw new BadRequestException(String.format("The combination of the parameters hl.snippet and hl.fragsize is too high (%d*%d). The maximum allowed value is %d.", snippets,fragsize, SolrKeys.MAX_HL_COMBINATION)); + } + } /** * Build XML document from SOLR response as a raw String From 2738b0535c9c643610ac5b1924e2fb91339e6ed0 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 2 Sep 2024 07:14:51 +0200 Subject: [PATCH 203/235] Issue #76 --- .../iterators/solr/SolrPageIteratorTest.java | 13 +- .../v60/redirection/ProxyHandlerSupport.java | 458 +++++++++--------- .../redirection/item/ProxyItemHandler.java | 4 + .../redirection/item/V5RedirectHandler.java | 36 +- .../redirection/item/V7RedirectHandler.java | 35 +- .../kramerius/rest/oai/MetadataExport.java | 325 +++++++------ .../incad/kramerius/rest/oai/OAIRecord.java | 80 ++- .../cz/incad/kramerius/rest/oai/OAISet.java | 103 +++- .../cz/incad/kramerius/rest/oai/OAITools.java | 78 +++ .../cz/incad/kramerius/rest/oai/OAIVerb.java | 93 +++- .../rest/oai/exceptions/OAIException.java | 16 +- 11 files changed, 813 insertions(+), 428 deletions(-) diff --git a/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/iterators/solr/SolrPageIteratorTest.java b/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/iterators/solr/SolrPageIteratorTest.java index d8ac852b29..7368d0261c 100644 --- a/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/iterators/solr/SolrPageIteratorTest.java +++ b/processes/cdkprocesses/src/test/java/cz/incad/kramerius/services/iterators/solr/SolrPageIteratorTest.java @@ -45,15 +45,15 @@ public void testPageIterator() throws IOException, SAXException, ParserConfigura Document document = XMLUtils.parseDocument(new StringReader(xmlConfig)); Client client = createMock(Client.class); - String firstReq = "http://solr_test:8983/test/select?q=*:*&start=0&rows=1000&fl=PID&wt=xml"; + String firstReq = "http://solr_test:8983/test/select?q=*:*&start=0&rows=1000&fl=PID&wt=xml&sort=PID+ASC"; String firstResp = IOUtils.toString(this.getClass().getResourceAsStream("pagination_first_page.xml"), "UTF-8"); List mockFromFirstCall = webCallExpect(client, firstReq, firstResp); - String secondReq = "http://solr_test:8983/test/select?q=*:*&start=1000&rows=1000&fl=PID&wt=xml"; + String secondReq = "http://solr_test:8983/test/select?q=*:*&start=1000&rows=1000&fl=PID&wt=xml&sort=PID+ASC"; String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("pagination_second_page.xml"), "UTF-8"); List mockFromSecondCall = webCallExpect(client, secondReq, secondResp); - String thirdReq = "http://solr_test:8983/test/select?q=*:*&start=2000&rows=1000&fl=PID&wt=xml"; + String thirdReq = "http://solr_test:8983/test/select?q=*:*&start=2000&rows=1000&fl=PID&wt=xml&sort=PID+ASC"; String thirdResp = IOUtils.toString(this.getClass().getResourceAsStream("pagination_third_page.xml"), "UTF-8"); List mockFromThirdCall = webCallExpect(client, thirdReq, thirdResp); @@ -101,15 +101,16 @@ public void testPageWithFilterIterator() throws IOException, SAXException, Parse Document document = XMLUtils.parseDocument(new StringReader(xmlConfig)); Client client = createMock(Client.class); - String firstReq = "http://solr_test:8983/test/select?q=*:*&start=0&rows=1000&fq=modified_date%3A%5B2016-05-03T00%3A16%3A19.440Z+TO+2017-05-03T04%3A16%3A19.440Z%5D&fl=PID&wt=xml"; + //http://solr_test:8983/test/select?q=*:*&start=0&rows=1000&fq=modified_date%3A%5B2016-05-03T00%3A16%3A19.440Z+TO+2017-05-03T04%3A16%3A19.440Z%5D&fl=PID&wt=xml&sort=PID+ASC + String firstReq = "http://solr_test:8983/test/select?q=*:*&start=0&rows=1000&fq=modified_date%3A%5B2016-05-03T00%3A16%3A19.440Z+TO+2017-05-03T04%3A16%3A19.440Z%5D&fl=PID&wt=xml&sort=PID+ASC"; String firstResp = IOUtils.toString(this.getClass().getResourceAsStream("pagination_first_page.xml"), "UTF-8"); List mockFromFirstCall = webCallExpect(client, firstReq, firstResp); - String secondReq = "http://solr_test:8983/test/select?q=*:*&start=1000&rows=1000&fq=modified_date%3A%5B2016-05-03T00%3A16%3A19.440Z+TO+2017-05-03T04%3A16%3A19.440Z%5D&fl=PID&wt=xml"; + String secondReq = "http://solr_test:8983/test/select?q=*:*&start=1000&rows=1000&fq=modified_date%3A%5B2016-05-03T00%3A16%3A19.440Z+TO+2017-05-03T04%3A16%3A19.440Z%5D&fl=PID&wt=xml&sort=PID+ASC"; String secondResp = IOUtils.toString(this.getClass().getResourceAsStream("pagination_second_page.xml"), "UTF-8"); List mockFromSecondCall = webCallExpect(client, secondReq, secondResp); - String thirdReq = "http://solr_test:8983/test/select?q=*:*&start=2000&rows=1000&fq=modified_date%3A%5B2016-05-03T00%3A16%3A19.440Z+TO+2017-05-03T04%3A16%3A19.440Z%5D&fl=PID&wt=xml"; + String thirdReq = "http://solr_test:8983/test/select?q=*:*&start=2000&rows=1000&fq=modified_date%3A%5B2016-05-03T00%3A16%3A19.440Z+TO+2017-05-03T04%3A16%3A19.440Z%5D&fl=PID&wt=xml&sort=PID+ASC"; String thirdResp = IOUtils.toString(this.getClass().getResourceAsStream("pagination_third_page.xml"), "UTF-8"); List mockFromThirdCall = webCallExpect(client, thirdReq, thirdResp); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 1c668172b8..d6ceebca0d 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -52,171 +52,171 @@ public abstract class ProxyHandlerSupport { - public static final boolean DEBUG = false; - - public static final Logger LOGGER = Logger.getLogger(ProxyHandlerSupport.class.getName()); - - protected String source; - protected Client client; - protected SolrAccess solrAccess; - protected User user; - protected String remoteAddr; - protected Instances instances; - protected ReharvestManager reharvestManager; - - public ProxyHandlerSupport(ReharvestManager reharvestManager, Instances instances, User user, Client client, SolrAccess solrAccess, String source, - String remoteAddr) { - this.reharvestManager = reharvestManager; - this.source = source; - this.client = client; - this.solrAccess = solrAccess; - this.user = user; - this.remoteAddr = remoteAddr; - this.instances = instances; - } - - public Response buildRedirectResponse(String url) throws ProxyHandlerException { - try { - LOGGER.info(String.format("Redirecting to %s", url)); - return Response.temporaryRedirect(new URL(url).toURI()).build(); - } catch (MalformedURLException | URISyntaxException e) { - throw new ProxyHandlerException(e); - } - } - - /** - * Build rewsponse with HEAD method - * @param url - * @return - * @throws ProxyHandlerException - */ - public Response buildForwardResponseHEAD(String url) throws ProxyHandlerException { - WebResource.Builder b = buidFowrardResponse(url); - ClientResponse clientResponseHead = b.head(); - if (clientResponseHead.getStatus() == 200) { - return Response.status(200).build(); - } else { - return Response.status(clientResponseHead.getStatus()).build(); - } - } - - public ClientResponse forwardedResponse(String url) throws ProxyHandlerException { - WebResource.Builder b = buidFowrardResponse(url); - ClientResponse response = b.get(ClientResponse.class); - if (response.getStatus() == 200) { - return response; - } else { - throw new ProxyHandlerException("Bad response; status code "+response.getStatus()); - } - } - + public static final boolean DEBUG = false; + + public static final Logger LOGGER = Logger.getLogger(ProxyHandlerSupport.class.getName()); + + protected String source; + protected Client client; + protected SolrAccess solrAccess; + protected User user; + protected String remoteAddr; + protected Instances instances; + protected ReharvestManager reharvestManager; + + public ProxyHandlerSupport(ReharvestManager reharvestManager, Instances instances, User user, Client client, + SolrAccess solrAccess, String source, String remoteAddr) { + this.reharvestManager = reharvestManager; + this.source = source; + this.client = client; + this.solrAccess = solrAccess; + this.user = user; + this.remoteAddr = remoteAddr; + this.instances = instances; + } + + public Response buildRedirectResponse(String url) throws ProxyHandlerException { + try { + LOGGER.info(String.format("Redirecting to %s", url)); + return Response.temporaryRedirect(new URL(url).toURI()).build(); + } catch (MalformedURLException | URISyntaxException e) { + throw new ProxyHandlerException(e); + } + } + + /** + * Build rewsponse with HEAD method + * + * @param url + * @return + * @throws ProxyHandlerException + */ + public Response buildForwardResponseHEAD(String url) throws ProxyHandlerException { + WebResource.Builder b = buidFowrardResponse(url); + ClientResponse clientResponseHead = b.head(); + if (clientResponseHead.getStatus() == 200) { + return Response.status(200).build(); + } else { + return Response.status(clientResponseHead.getStatus()).build(); + } + } + + public ClientResponse forwardedResponse(String url) throws ProxyHandlerException { + WebResource.Builder b = buidFowrardResponse(url); + ClientResponse response = b.get(ClientResponse.class); + if (response.getStatus() == 200) { + return response; + } else { + throw new ProxyHandlerException("Bad response; status code " + response.getStatus()); + } + } + public Response buildForwardResponseGET(String url, boolean deleteTrigger) throws ProxyHandlerException { return buildForwardResponseGET(url, null, null, deleteTrigger); } - - public Response buildForwardResponseGET(String url, String pid, boolean deleteTrigger) throws ProxyHandlerException { + + public Response buildForwardResponseGET(String url, String pid, boolean deleteTrigger) + throws ProxyHandlerException { return buildForwardResponseGET(url, null, pid, deleteTrigger); } - - - public Response buildForwardResponseGET(String url, String mimetype, String pid, boolean deleteTrigger) throws ProxyHandlerException { - WebResource.Builder b = buidFowrardResponse(url); - ClientResponse response = b.get(ClientResponse.class); - if (response.getStatus() == 200) { - String responseMimeType = response.getType().toString(); - InputStream is = response.getEntityInputStream(); - MultivaluedMap headers = response.getHeaders(); - - StreamingOutput stream = new StreamingOutput() { - public void write(OutputStream output) throws IOException, WebApplicationException { - try { - IOUtils.copy(is, output); - } catch (Exception e) { - throw new WebApplicationException(e); - } - } - }; - ResponseBuilder respEntity = null; - if (mimetype != null) { + + public Response buildForwardResponseGET(String url, String mimetype, String pid, boolean deleteTrigger) + throws ProxyHandlerException { + WebResource.Builder b = buidFowrardResponse(url); + ClientResponse response = b.get(ClientResponse.class); + if (response.getStatus() == 200) { + String responseMimeType = response.getType().toString(); + InputStream is = response.getEntityInputStream(); + MultivaluedMap headers = response.getHeaders(); + + StreamingOutput stream = new StreamingOutput() { + public void write(OutputStream output) throws IOException, WebApplicationException { + try { + IOUtils.copy(is, output); + } catch (Exception e) { + throw new WebApplicationException(e); + } + } + }; + ResponseBuilder respEntity = null; + if (mimetype != null) { respEntity = Response.status(200).entity(stream).type(mimetype); - } else if (responseMimeType != null) { + } else if (responseMimeType != null) { respEntity = Response.status(200).entity(stream).type(responseMimeType); - } else { - respEntity = Response.status(200).entity(stream); - } - - /* Disable header forward - headers.keySet().forEach(key -> { - List values = headers.get(key); - values.stream().forEach(val -> { - respEntity.header(key, val); - }); - }); - */ - - return respEntity.build(); - } else { + } else { + respEntity = Response.status(200).entity(stream); + } + + /* + * Disable header forward headers.keySet().forEach(key -> { List values + * = headers.get(key); values.stream().forEach(val -> { respEntity.header(key, + * val); }); }); + */ + + return respEntity.build(); + } else { // event for reharvest - if (response.getStatus() == 404) { - if (deleteTrigger) deleteTriggeToReharvest(pid); - } - return Response.status(response.getStatus()).build(); - } - } + if (response.getStatus() == 404) { + if (deleteTrigger) + deleteTriggeToReharvest(pid); + } + return Response.status(response.getStatus()).build(); + } + } public void deleteTriggeToReharvest(String pid) { if (reharvestManager != null && pid != null) { - //LOGGER.info(String.format("Registering pid %s",pid)); + // LOGGER.info(String.format("Registering pid %s",pid)); try { Document solrDataByPid = this.solrAccess.getSolrDataByPid(pid); - Element rootPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - if (element.getNodeName().equals("str")) { - String fieldName = element.getAttribute("name"); - return fieldName.equals("root.pid"); - } - return false; - } - }); - - Element ownPidPath = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - if (element.getNodeName().equals("str")) { - String fieldName = element.getAttribute("name"); - return fieldName.equals("own_pid_path"); - } - return false; - } - }); - - Element ownParentPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - if (element.getNodeName().equals("str")) { - String fieldName = element.getAttribute("name"); - return fieldName.equals("own_parent.pid"); - } - return false; - } - }); - - - Element cdkCollection = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - if (element.getNodeName().equals("arr")) { - String fieldName = element.getAttribute("name"); - return fieldName.equals("cdk.collection"); - } - return false; - } - }); + Element rootPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), + new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("str")) { + String fieldName = element.getAttribute("name"); + return fieldName.equals("root.pid"); + } + return false; + } + }); + + Element ownPidPath = XMLUtils.findElement(solrDataByPid.getDocumentElement(), + new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("str")) { + String fieldName = element.getAttribute("name"); + return fieldName.equals("own_pid_path"); + } + return false; + } + }); + + Element ownParentPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), + new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("str")) { + String fieldName = element.getAttribute("name"); + return fieldName.equals("own_parent.pid"); + } + return false; + } + }); + + Element cdkCollection = XMLUtils.findElement(solrDataByPid.getDocumentElement(), + new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("arr")) { + String fieldName = element.getAttribute("name"); + return fieldName.equals("cdk.collection"); + } + return false; + } + }); - - - if (rootPid != null && ownPidPath != null && ownParentPid != null) { String pidPath = ownPidPath.getTextContent().trim(); String ownParentPidText = ownParentPid.getTextContent().trim(); @@ -225,9 +225,11 @@ public boolean acceptElement(Element element) { pidPath = pidPath.substring(0, index + ownParentPidText.length()).trim(); } try { - ReharvestItem alreadyRegistredItem = this.reharvestManager.getOpenItemByPid(ownParentPid.getTextContent().trim()); + ReharvestItem alreadyRegistredItem = this.reharvestManager + .getOpenItemByPid(ownParentPid.getTextContent().trim()); if (alreadyRegistredItem == null) { - ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger|404 ","open", ownParentPid.getTextContent().trim(), pidPath); + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), + "Delete trigger|404 ", "open", ownParentPid.getTextContent().trim(), pidPath); reharvestItem.setTypeOfReharvest(TypeOfReharvset.children); reharvestItem.setState("waiting_for_approve"); if (cdkCollection != null) { @@ -236,97 +238,103 @@ public boolean acceptElement(Element element) { List enabledInstances = this.instances.enabledInstances(); for (OneInstance inst : enabledInstances) { String acronym = inst.getName(); - boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + acronym + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + acronym + ".licenses") : false; - if (channelAccess) { collections.add(acronym); } + boolean channelAccess = KConfiguration.getInstance().getConfiguration() + .containsKey("cdk.collections.sources." + acronym + ".licenses") + ? KConfiguration.getInstance().getConfiguration().getBoolean( + "cdk.collections.sources." + acronym + ".licenses") + : false; + if (channelAccess) { + collections.add(acronym); + } } reharvestItem.setLibraries(collections); } this.reharvestManager.register(reharvestItem); } - + } catch (DOMException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); + LOGGER.log(Level.SEVERE, e.getMessage(), e); } catch (AlreadyRegistedPidsException e) { - LOGGER.log(Level.SEVERE,e.getMessage(),e); + LOGGER.log(Level.SEVERE, e.getMessage(), e); } } else { LOGGER.log(Level.SEVERE, "Cannot find root.pid element"); } } catch (IOException e) { - LOGGER.log(Level.SEVERE,e.getMessage()); + LOGGER.log(Level.SEVERE, e.getMessage()); } } else { - LOGGER.log(Level.SEVERE,"No reharvest manager or pid "); + LOGGER.log(Level.SEVERE, "No reharvest manager or pid "); } } - protected void mockSession() { - if (!user.getSessionAttributes().containsKey("shib-session-id")) { - this.user.addSessionAttribute("shib-session-id", "_dd68cbd66641c9b647b05509ac0241fa"); - } - if (!user.getSessionAttributes().containsKey("shib-session-expires")) { - this.user.addSessionAttribute("shib-session-expires", "1592847906"); - } - if (!user.getSessionAttributes().containsKey("shib-identity-provider")) { - this.user.addSessionAttribute("shib-identity-provider", - "https://shibboleth.mzk.cz/simplesaml/metadata.xml"); - } - if (!user.getSessionAttributes().containsKey("shib-authentication-method")) { - this.user.addSessionAttribute("shib-authentication-method", - "urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport"); - } - - if (!user.getSessionAttributes().containsKey("shib-handler")) { - this.user.addSessionAttribute("shib-handler", "https://dnnt.mzk.cz/Shibboleth.sso"); - } - if (!user.getSessionAttributes().containsKey("remote_user")) { - this.user.addSessionAttribute("remote_user", "all_users@mzk.cz"); - } - if (!user.getSessionAttributes().containsKey("affiliation")) { - this.user.addSessionAttribute("affiliation", "member@mzk.cz"); - } - if (!user.getSessionAttributes().containsKey("entitlement")) { - this.user.addSessionAttribute("entitlement", "cokoliv"); - } - if (!user.getSessionAttributes().containsKey("edupersonuniqueid")) { - this.user.addSessionAttribute("edupersonuniqueid", "user@mzk.cz"); - } - } - - protected WebResource.Builder buidFowrardResponse(String url) { - String prefixHeaders = KConfiguration.getInstance().getConfiguration().getString("cdk.shibboleth.forward.headers"); - - // no user session attributes in case of no federation - String header = ""; + protected void mockSession() { + if (!user.getSessionAttributes().containsKey("shib-session-id")) { + this.user.addSessionAttribute("shib-session-id", "_dd68cbd66641c9b647b05509ac0241fa"); + } + if (!user.getSessionAttributes().containsKey("shib-session-expires")) { + this.user.addSessionAttribute("shib-session-expires", "1592847906"); + } + if (!user.getSessionAttributes().containsKey("shib-identity-provider")) { + this.user.addSessionAttribute("shib-identity-provider", + "https://shibboleth.mzk.cz/simplesaml/metadata.xml"); + } + if (!user.getSessionAttributes().containsKey("shib-authentication-method")) { + this.user.addSessionAttribute("shib-authentication-method", + "urn:oasis:names:tc:SAML:2.0:ac:classes:PasswordProtectedTransport"); + } + + if (!user.getSessionAttributes().containsKey("shib-handler")) { + this.user.addSessionAttribute("shib-handler", "https://dnnt.mzk.cz/Shibboleth.sso"); + } + if (!user.getSessionAttributes().containsKey("remote_user")) { + this.user.addSessionAttribute("remote_user", "all_users@mzk.cz"); + } + if (!user.getSessionAttributes().containsKey("affiliation")) { + this.user.addSessionAttribute("affiliation", "member@mzk.cz"); + } + if (!user.getSessionAttributes().containsKey("entitlement")) { + this.user.addSessionAttribute("entitlement", "cokoliv"); + } + if (!user.getSessionAttributes().containsKey("edupersonuniqueid")) { + this.user.addSessionAttribute("edupersonuniqueid", "user@mzk.cz"); + } + } + + protected WebResource.Builder buidFowrardResponse(String url) { + String prefixHeaders = KConfiguration.getInstance().getConfiguration() + .getString("cdk.shibboleth.forward.headers"); + + // no user session attributes in case of no federation + String header = ""; boolean shibbolethAttributes = KConfiguration.getInstance().getConfiguration() .getBoolean("cdk.collections.sources." + this.source + ".shibboleth_attributes", true); - - if (shibbolethAttributes) { - Map attributes = this.user.getSessionAttributes(); - header = header + attributes.keySet().stream().map(key -> { - return "header_" + key + "=" + attributes.get(key); - }).collect(Collectors.joining("|")); - } - - if (this.remoteAddr != null) { - header = header + "|" + "header_ip_address=" + this.remoteAddr; - } - //TODO: Source - if (StringUtils.isAnyString(prefixHeaders)) { - header = prefixHeaders+header; - } - - LOGGER.fine(String.format("Requesting %s", url)); - WebResource r = client.resource(url); - LOGGER.info("CDK_TOKEN_PARAMETERS = "+header); - return r.header("CDK_TOKEN_PARAMETERS", header); - } - - protected String baseUrl() { - String baseurl = KConfiguration.getInstance().getConfiguration() - .getString("cdk.collections.sources." + this.source + ".baseurl"); - return baseurl; - } + if (shibbolethAttributes) { + Map attributes = this.user.getSessionAttributes(); + header = header + attributes.keySet().stream().map(key -> { + return "header_" + key + "=" + attributes.get(key); + }).collect(Collectors.joining("|")); + } + + if (this.remoteAddr != null) { + header = header + "|" + "header_ip_address=" + this.remoteAddr; + } + // TODO: Source + if (StringUtils.isAnyString(prefixHeaders)) { + header = prefixHeaders + header; + } + + LOGGER.fine(String.format("Requesting %s", url)); + WebResource r = client.resource(url); + LOGGER.info("CDK_TOKEN_PARAMETERS = " + header); + return r.header("CDK_TOKEN_PARAMETERS", header); + } + + protected String baseUrl() { + String baseurl = KConfiguration.getInstance().getConfiguration() + .getString("cdk.collections.sources." + this.source + ".baseurl"); + return baseurl; + } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java index 01d1d84e03..f6a61ea04a 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/ProxyItemHandler.java @@ -77,6 +77,10 @@ public ProxyItemHandler(ReharvestManager reharvestManager, Instances instances, // helper methods for direct access to dc stream public abstract InputStream directStreamDC() throws ProxyHandlerException; + public abstract InputStream directStreamBiblioMods() throws ProxyHandlerException; + + public abstract boolean isStreamDCAvaiable() throws ProxyHandlerException; + public abstract boolean isStreamBiblioModsAvaiable() throws ProxyHandlerException; public boolean imageThumbForceRedirection() { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java index 8dad1db896..9cc784c54e 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5RedirectHandler.java @@ -101,20 +101,52 @@ public Response dc(RequestMethodName method) throws ProxyHandlerException { return buildRedirectResponse(url); } + + @Override + public InputStream directStreamDC() throws ProxyHandlerException { + return directStream("DC"); + } + @Override - public InputStream directStreamDC() throws ProxyHandlerException { + public InputStream directStreamBiblioMods() throws ProxyHandlerException { + return directStream("BIBLIO_MODS"); + } + + + private InputStream directStream(String stream) throws ProxyHandlerException { String baseurl = super.baseUrl(); - String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/item/" + this.pid + "/streams/DC"; + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/item/" + this.pid + "/streams/"+stream; WebResource.Builder b = buidFowrardResponse(url); ClientResponse response = b.get(ClientResponse.class); if (response.getStatus() == 200) { InputStream is = response.getEntityInputStream(); return is; } else return null; + + } + + private boolean isStreamAvailable(String stream) throws ProxyHandlerException { + String baseurl = super.baseUrl(); + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/item/" + this.pid + "/streams/"+stream; + WebResource.Builder b = buidFowrardResponse(url); + ClientResponse response = b.head(); + if (response.getStatus() == 200) { + return true; + } else return false; } + @Override + public boolean isStreamDCAvaiable() throws ProxyHandlerException { + return isStreamAvailable("DC"); + } + + @Override + public boolean isStreamBiblioModsAvaiable() throws ProxyHandlerException { + return isStreamAvailable("BIBLIO_MODS"); + } + @Override public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHandlerException { String baseurl = baseUrl(); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java index 5204b26c1d..cf62165882 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7RedirectHandler.java @@ -192,8 +192,21 @@ public Response audioWAV() throws ProxyHandlerException { @Override public InputStream directStreamDC() throws ProxyHandlerException { + return directStream("dc"); + } + + + + @Override + public InputStream directStreamBiblioMods() throws ProxyHandlerException { + return directStream("mods"); + } + + + // TODO: Consider if this should be + private InputStream directStream(String stream) { String baseurl = baseUrl(); - String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/client/v7.0/items/" + this.pid + "/metadata/dc"; + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/client/v7.0/items/" + this.pid + "/metadata/"+stream; WebResource.Builder b = buidFowrardResponse(url); ClientResponse response = b.get(ClientResponse.class); if (response.getStatus() == 200) { @@ -203,6 +216,26 @@ public InputStream directStreamDC() throws ProxyHandlerException { } + private boolean isStreamAvailable(String stream) throws ProxyHandlerException { + String baseurl = super.baseUrl(); + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/client/v7.0/items/" + this.pid + "/metadata/"+stream; + WebResource.Builder b = buidFowrardResponse(url); + ClientResponse response = b.head(); + if (response.getStatus() == 200) { + return true; + } else return false; + } + + @Override + public boolean isStreamDCAvaiable() throws ProxyHandlerException { + return isStreamAvailable("dc"); + } + + @Override + public boolean isStreamBiblioModsAvaiable() throws ProxyHandlerException { + return isStreamAvailable("mods"); + } + @Override public Response iiifInfo(RequestMethodName method, String pid) throws ProxyHandlerException { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java index a02153b17e..9015c27053 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/MetadataExport.java @@ -57,13 +57,17 @@ public enum MetadataExport { public Element performOnCDKSide(SolrAccess solrAccess,Provider userProvider, Provider clientProvider, Instances instances, HttpServletRequest request, Document owningDocument, OAIRecord oaiRec,OAISet set) { try { String pid = OAITools.pidFromOAIIdentifier(oaiRec.getIdentifier()); - InputStream directStreamDC = dcStream(solrAccess, userProvider, clientProvider, instances, - request, pid); - if (directStreamDC != null) { - Document dc = XMLUtils.parseDocument(directStreamDC, true); - Element rootElement = dc.getDocumentElement(); - owningDocument.adoptNode(rootElement); - return rootElement; + ProxyItemHandler redirectHandler = findRedirectHandler(solrAccess, userProvider, clientProvider, instances, request, pid, null); + if (redirectHandler != null) { + InputStream directStreamDC = redirectHandler.directStreamDC(); + if (directStreamDC != null) { + Document dc = XMLUtils.parseDocument(directStreamDC, true); + Element rootElement = dc.getDocumentElement(); + owningDocument.adoptNode(rootElement); + return rootElement; + } else { + return null; + } } else { return null; } @@ -85,102 +89,107 @@ public Element performOnCDKSide(SolrAccess solrAccess,Provider userProvide String baseUrl = ApplicationURL.applicationURL(request); //rdf:about="uuid:6b182ad3-b9e9-11e1-1726-001143e3f55c" String pid = OAITools.pidFromOAIIdentifier(oaiRec.getIdentifier()); - InputStream directStreamDC = dcStream(solrAccess, userProvider, clientProvider, instances, - request, pid); - if (directStreamDC != null) { - Document dc = XMLUtils.parseDocument(directStreamDC, true); - Element dcElement = dc.getDocumentElement(); - - Element metadata = owningDocument.createElement("metadata"); - metadata.setAttribute("xmlns:europeana", "http://www.europeana.eu/schemas/ese/"); - metadata.setAttribute("xmlns:ore", "http://www.openarchives.org/ore/terms/"); - metadata.setAttribute("xmlns:edm", "http://www.europeana.eu/schemas/edm/"); - metadata.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); - metadata.setAttribute("xmlns:rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#"); - metadata.setAttribute("xmlns:rdaGr2", "http://rdvocab.info/ElementsGr2/"); - metadata.setAttribute("xmlns:skos", "http://www.w3.org/2004/02/skos/core#"); - metadata.setAttribute("xmlns:oai_dc", "http://www.openarchives.org/OAI/2.0/oai_dc/"); - metadata.setAttribute("xmlns:dc", "http://purl.org/dc/elements/1.1/"); - metadata.setAttribute("xmlns:dcterms", "http://purl.org/dc/terms/"); - - Element rdf = owningDocument.createElementNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#","rdf:RDF"); - rdf.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",oaiRec.getIdentifier()); - metadata.appendChild(rdf); - Element providedCHO = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:ProvidedCHO"); - rdf.appendChild(providedCHO); - - List elements = XMLUtils.getElements(dcElement); - elements.stream().forEach(dcElm -> { - owningDocument.adoptNode(dcElm); - providedCHO.appendChild(dcElm); - }); - - //rdf.appendChild(providedCHO); - - Element type = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:type"); - providedCHO.appendChild(type); - type.setTextContent("TEXT"); - - Element webresource = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:WebResource"); - webresource.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",String.format("%s/api/client/v7.0/items/%s/image", baseUrl, pid)); - metadata.appendChild(webresource); - - Element edmAggregation = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:Aggregation"); - String clientUrl = KConfiguration.getInstance().getConfiguration().getString("client"); - if (clientUrl != null) { - edmAggregation.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); - } else { - edmAggregation.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); - } - Element edmDataPrvovider = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:dataProvider"); - - // Data provider - // find data provider by acronym - String acronym = KConfiguration.getInstance().getConfiguration().getString("acronym",""); - String defaultEDMDataProvider = KConfiguration.getInstance().getConfiguration().getString("oai.set.edm.dataProvider",acronym); -// if (edmDataProvider != null) { -// edmDataPrvovider.setTextContent(edmDataProvider); -// } - metadataProvider(instances, oaiRec, edmDataPrvovider, defaultEDMDataProvider); - edmAggregation.appendChild(edmDataPrvovider); - - // dodat dle setu - Element shownAt = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:isShownAt"); - if (clientUrl != null) { - shownAt.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource",clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); - } else { - shownAt.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource",baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); - } - edmAggregation.appendChild(shownAt); - - // mapovani na licence - Element edmRights = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:rights"); + ProxyItemHandler redirectHandler = findRedirectHandler(solrAccess, userProvider, clientProvider, instances, request, pid, null); + if (redirectHandler != null) { + InputStream directStreamDC = redirectHandler.directStreamDC(); + if (directStreamDC != null) { + Document dc = XMLUtils.parseDocument(directStreamDC, true); + Element dcElement = dc.getDocumentElement(); + + Element metadata = owningDocument.createElement("metadata"); + metadata.setAttribute("xmlns:europeana", "http://www.europeana.eu/schemas/ese/"); + metadata.setAttribute("xmlns:ore", "http://www.openarchives.org/ore/terms/"); + metadata.setAttribute("xmlns:edm", "http://www.europeana.eu/schemas/edm/"); + metadata.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance"); + metadata.setAttribute("xmlns:rdf", "http://www.w3.org/1999/02/22-rdf-syntax-ns#"); + metadata.setAttribute("xmlns:rdaGr2", "http://rdvocab.info/ElementsGr2/"); + metadata.setAttribute("xmlns:skos", "http://www.w3.org/2004/02/skos/core#"); + metadata.setAttribute("xmlns:oai_dc", "http://www.openarchives.org/OAI/2.0/oai_dc/"); + metadata.setAttribute("xmlns:dc", "http://purl.org/dc/elements/1.1/"); + metadata.setAttribute("xmlns:dcterms", "http://purl.org/dc/terms/"); + + Element rdf = owningDocument.createElementNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#","rdf:RDF"); + rdf.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",oaiRec.getIdentifier()); + metadata.appendChild(rdf); - if (clientUrl != null) { - edmRights.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource", clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); - } else { - edmRights.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource", baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); - } + Element providedCHO = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:ProvidedCHO"); + rdf.appendChild(providedCHO); + + List elements = XMLUtils.getElements(dcElement); + elements.stream().forEach(dcElm -> { + owningDocument.adoptNode(dcElm); + providedCHO.appendChild(dcElm); + }); + + //rdf.appendChild(providedCHO); + + Element type = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:type"); + providedCHO.appendChild(type); + type.setTextContent("TEXT"); + + Element webresource = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:WebResource"); + webresource.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",String.format("%s/api/client/v7.0/items/%s/image", baseUrl, pid)); + metadata.appendChild(webresource); + + Element edmAggregation = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:Aggregation"); + String clientUrl = KConfiguration.getInstance().getConfiguration().getString("client"); + if (clientUrl != null) { + edmAggregation.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); + } else { + edmAggregation.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:about",baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); + } + Element edmDataPrvovider = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:dataProvider"); + + // Data provider + // find data provider by acronym + String acronym = KConfiguration.getInstance().getConfiguration().getString("acronym",""); + String defaultEDMDataProvider = KConfiguration.getInstance().getConfiguration().getString("oai.set.edm.dataProvider",acronym); +// if (edmDataProvider != null) { +// edmDataPrvovider.setTextContent(edmDataProvider); +// } + metadataProvider(instances, oaiRec, edmDataPrvovider, defaultEDMDataProvider); + edmAggregation.appendChild(edmDataPrvovider); + + // dodat dle setu + Element shownAt = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:isShownAt"); + if (clientUrl != null) { + shownAt.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource",clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); + } else { + shownAt.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource",baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); + } + edmAggregation.appendChild(shownAt); + + // mapovani na licence + Element edmRights = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:rights"); + + if (clientUrl != null) { + edmRights.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource", clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); + } else { + edmRights.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource", baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); + } - edmAggregation.appendChild(edmRights); - - - Element edmObject = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:object"); - edmObject.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource", String.format("%s/api/client/v7.0/items/%s/image", baseUrl, pid)); - edmAggregation.appendChild(edmObject); - - - // ceska digitalni kniovna - Element edmProvider = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:provider"); - String edmProviderText = KConfiguration.getInstance().getConfiguration().getString("oai.set.edm.provider",acronym); - edmProvider.setTextContent( edmProviderText); //"Czech digital library/Česká digitální knihovna"); - edmAggregation.appendChild(edmProvider); - - metadata.appendChild(edmAggregation); - - return metadata; + edmAggregation.appendChild(edmRights); + + + Element edmObject = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:object"); + edmObject.setAttributeNS("http://www.w3.org/1999/02/22-rdf-syntax-ns#", "rdf:resource", String.format("%s/api/client/v7.0/items/%s/image", baseUrl, pid)); + edmAggregation.appendChild(edmObject); + + + // ceska digitalni kniovna + Element edmProvider = owningDocument.createElementNS("http://www.europeana.eu/schemas/edm/","edm:provider"); + String edmProviderText = KConfiguration.getInstance().getConfiguration().getString("oai.set.edm.provider",acronym); + edmProvider.setTextContent( edmProviderText); //"Czech digital library/Česká digitální knihovna"); + edmAggregation.appendChild(edmProvider); + + metadata.appendChild(edmAggregation); + + return metadata; + } else { + return null; + } } else { return null; } @@ -198,60 +207,61 @@ public Element performOnCDKSide(SolrAccess solrAccess,Provider userProvide "http://www.europeana.eu/schemas/ese/ESE-V3.2.xsd", "http://www.europeana.eu/schemas/ese/") { - - @Override public Element performOnCDKSide(SolrAccess solrAccess,Provider userProvider, Provider clientProvider, Instances instances, HttpServletRequest request, Document owningDocument, OAIRecord oaiRec,OAISet set) { try { String baseUrl = ApplicationURL.applicationURL(request); String pid = OAITools.pidFromOAIIdentifier(oaiRec.getIdentifier()); + ProxyItemHandler redirectHandler = findRedirectHandler(solrAccess, userProvider, clientProvider, instances, request, pid, null); + if (redirectHandler != null) { + InputStream directStreamDC = redirectHandler.directStreamDC(); + if (directStreamDC != null) { + Document dc = XMLUtils.parseDocument(directStreamDC, true); + Element dcElement = dc.getDocumentElement(); - InputStream directStreamDC = dcStream(solrAccess, userProvider, clientProvider, instances, - request,pid); - if (directStreamDC != null) { - Document dc = XMLUtils.parseDocument(directStreamDC, true); - Element dcElement = dc.getDocumentElement(); + Element record = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "record"); + record.setAttribute("xmlns:dc", FedoraNamespaces.DC_NAMESPACE_URI); + + record.setAttributeNS("http://www.w3.org/2001/XMLSchema-instance","xsi:schemaLocation","http://www.europeana.eu/schemas/ese/ http://www.europeana.eu/schemas/ese/ESE-V3.2.xsd http://purl.org/dc/elements/1.1/ http://www.dublincore.org/schemas/xmls/qdc/dc.xsd http://purl.org/dc/terms/ http://www.dublincore.org/schemas/xmls/qdc/dcterms.xsd http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd"); + List dcElems = XMLUtils.getElements(dcElement); + dcElems.stream().forEach(dcElm-> { + owningDocument.adoptNode(dcElm); + record.appendChild(dcElm); + }); + + Element object = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "object"); + String thumb = String.format(baseUrl+(baseUrl.endsWith("/")? "" : "/")+"api/client/v7.0/items/%s/image/thumb", pid); + object.setTextContent(thumb); + record.appendChild(object); + + Element provider = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "provider"); + String acronym = KConfiguration.getInstance().getConfiguration().getString("acronym",""); + String defaultDataProvider = KConfiguration.getInstance().getConfiguration().getString("oai.set.edm.dataProvider",acronym); - Element record = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "record"); - record.setAttribute("xmlns:dc", FedoraNamespaces.DC_NAMESPACE_URI); - - record.setAttributeNS("http://www.w3.org/2001/XMLSchema-instance","xsi:schemaLocation","http://www.europeana.eu/schemas/ese/ http://www.europeana.eu/schemas/ese/ESE-V3.2.xsd http://purl.org/dc/elements/1.1/ http://www.dublincore.org/schemas/xmls/qdc/dc.xsd http://purl.org/dc/terms/ http://www.dublincore.org/schemas/xmls/qdc/dcterms.xsd http://www.openarchives.org/OAI/2.0/oai_dc/ http://www.openarchives.org/OAI/2.0/oai_dc.xsd"); - List dcElems = XMLUtils.getElements(dcElement); - dcElems.stream().forEach(dcElm-> { - owningDocument.adoptNode(dcElm); - record.appendChild(dcElm); - }); + metadataProvider(instances, oaiRec, provider, defaultDataProvider); + + record.appendChild(provider); + + Element type = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "type"); + type.setTextContent("TEXT"); + record.appendChild(type); + + Element isShownAt = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "isShownAt"); + String clientUrl = KConfiguration.getInstance().getConfiguration().getString("client"); + if (clientUrl != null) { + isShownAt.setTextContent(clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); + } else { + isShownAt.setTextContent(baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); + } + record.appendChild(isShownAt); + + + return record; + } else return null; - Element object = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "object"); - String thumb = String.format(baseUrl+(baseUrl.endsWith("/")? "" : "/")+"api/client/v7.0/items/%s/image/thumb", pid); - object.setTextContent(thumb); - record.appendChild(object); - - Element provider = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "provider"); - String acronym = KConfiguration.getInstance().getConfiguration().getString("acronym",""); - String defaultDataProvider = KConfiguration.getInstance().getConfiguration().getString("oai.set.edm.dataProvider",acronym); - - metadataProvider(instances, oaiRec, provider, defaultDataProvider); - - record.appendChild(provider); - - Element type = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "type"); - type.setTextContent("TEXT"); - record.appendChild(type); - - Element isShownAt = owningDocument.createElementNS("http://www.europeana.eu/schemas/ese/", "isShownAt"); - String clientUrl = KConfiguration.getInstance().getConfiguration().getString("client"); - if (clientUrl != null) { - isShownAt.setTextContent(clientUrl+(clientUrl.endsWith("/") ? "" : "/")+"uuid/"+pid); - } else { - isShownAt.setTextContent(baseUrl+(baseUrl.endsWith("/") ? "" : "/")+"/uuid/"+pid); - } - record.appendChild(isShownAt); - - - return record; - } else return null; - + } else { + return null; + } } catch (IOException | LexerException | ProxyHandlerException | ParserConfigurationException | SAXException e) { LOGGER.log(Level.SEVERE,e.getMessage(),e); throw new RuntimeException(e.getMessage()); @@ -300,22 +310,16 @@ public static MetadataExport findByPrefix(String prefix) { return null; } - private static InputStream dcStream(SolrAccess solrAccess, Provider userProvider, - Provider clientProvider, Instances instances, HttpServletRequest request, String pid) - throws LexerException, IOException, ProxyHandlerException { - ProxyItemHandler redirectHandler = findRedirectHandler(solrAccess, userProvider, clientProvider, instances, request, pid, null); - InputStream directStreamDC = redirectHandler.directStreamDC(); - return directStreamDC; - } + //TODO: Remove +// private static InputStream dcStream(SolrAccess solrAccess, Provider userProvider, +// Provider clientProvider, Instances instances, HttpServletRequest request, String pid) +// throws LexerException, IOException, ProxyHandlerException { +// ProxyItemHandler redirectHandler = findRedirectHandler(solrAccess, userProvider, clientProvider, instances, request, pid, null); +// InputStream directStreamDC = redirectHandler.directStreamDC(); +// return directStreamDC; +// } + - public Document getDC(ProxyItemHandler handler) throws ProxyHandlerException, ParserConfigurationException, SAXException, IOException { - InputStream is = handler.directStreamDC(); - if (is != null) { - Document document = XMLUtils.parseDocument(is, true); - return document; - } else return null; - - } private String metadataPrefix; private String schema; @@ -332,6 +336,7 @@ public static ProxyItemHandler findRedirectHandler(SolrAccess solrAccess,Provide if (source == null) { source = defaultDocumentSource(solrAccess, pid); } + LOGGER.info("Info - default source "+source); OneInstance found = instances.find(source); if (found!= null) { String remoteAddress = IPAddressUtils.getRemoteAddress(request, KConfiguration.getInstance().getConfiguration()); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java index b076164aa9..9001372854 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIRecord.java @@ -17,18 +17,22 @@ package cz.incad.kramerius.rest.oai; import java.io.IOException; +import java.io.InputStream; import java.io.StringReader; import java.net.URLEncoder; import java.time.OffsetDateTime; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import java.util.stream.Collectors; import javax.inject.Provider; import javax.servlet.http.HttpServletRequest; import javax.xml.parsers.ParserConfigurationException; +import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; import org.xml.sax.SAXException; @@ -39,6 +43,7 @@ import cz.incad.kramerius.SolrAccess; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; +import cz.incad.kramerius.rest.apiNew.client.v60.redirection.ProxyHandlerException; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.item.ProxyItemHandler; import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.IPAddressUtils; @@ -48,6 +53,8 @@ public class OAIRecord { + public static final Logger LOGGER = Logger.getLogger(OAIRecord.class.getName()); + private String identifier; private String solrIdentifier; @@ -55,10 +62,14 @@ public class OAIRecord { private List cdkCollections = new ArrayList<>(); - public OAIRecord(String solrIdentifier, String identifier) { + private String dateTimeStamp; + + + public OAIRecord(String solrIdentifier, String identifier, String dateTimeStamp) { super(); this.solrIdentifier = solrIdentifier; this.identifier = identifier; + this.dateTimeStamp = dateTimeStamp; } public String getIdentifier() { @@ -123,7 +134,16 @@ public boolean acceptElement(Element element) { } }); - OAIRecord oaiRecord = new OAIRecord(pidElm.getTextContent(), oaiIdentifier); + Element dateElm = XMLUtils.findElement(doc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + return name.equals("indexed"); + + } + }); + + OAIRecord oaiRecord = new OAIRecord(pidElm.getTextContent(), oaiIdentifier, dateElm != null ? dateElm.getTextContent() : ""); oaiRecord.setCdkCollections(collections.stream().map(Element::getTextContent).collect(Collectors.toList())); return oaiRecord; @@ -138,26 +158,70 @@ public boolean acceptElement(Element element) { public Element toMetadata(SolrAccess solrAccess,Provider userProvider, Provider clientProvider, Instances instances, HttpServletRequest request, Document owningDocument, String oaiIdentifier, MetadataExport export, OAISet set) { return export.performOnCDKSide(solrAccess,userProvider, clientProvider, instances, request, owningDocument, this, set); } -// +// +// /** render header +// * @throws IOException */ +// public Element toHeader(Document doc, FedoraAccess fa, OAISet set ) throws IOException { +// +// Element header = doc.createElement("header"); +// Element identifier = doc.createElement("identifier"); +// identifier.setTextContent(this.identifier); +// header.appendChild(identifier); +// +// //OffsetDateTime now = OffsetDateTime.now(); +// Element datestamp = doc.createElement("datestamp"); +// datestamp.setTextContent(this.dateTimeStamp); +// header.appendChild(datestamp); +// +// if (set != null) { +// Element setSpecElm = doc.createElement("setSpec"); +// setSpecElm.setTextContent(set.getSetSpec()); +// header.appendChild(setSpecElm); +// } +// +// String pid = OAITools.pidFromOAIIdentifier(this.identifier); +// if (!fa.isObjectAvailable(pid) && (!pid.contains("_"))) { +// header.setAttribute("status", "deleted"); +// } +// return header; +// } + + /** render header */ - public Element toHeader(Document doc, String setSpec) { + public Element toHeader(Document doc, OAISet set, SolrAccess solrAccess,Provider userProvider, Provider clientProvider, Instances instances, HttpServletRequest request, String source) { + Element header = doc.createElement("header"); Element identifier = doc.createElement("identifier"); identifier.setTextContent(this.identifier); header.appendChild(identifier); - OffsetDateTime now = OffsetDateTime.now(); + //OffsetDateTime now = OffsetDateTime.now(); Element datestamp = doc.createElement("datestamp"); - datestamp.setTextContent(now.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); header.appendChild(datestamp); + datestamp.setTextContent(this.dateTimeStamp); - if (setSpec != null) { + if (set != null) { Element setSpecElm = doc.createElement("setSpec"); - setSpecElm.setTextContent(setSpec); + setSpecElm.setTextContent(set.getSetSpec()); header.appendChild(setSpecElm); } + try { + String pid = OAITools.pidFromOAIIdentifier(this.identifier); + ProxyItemHandler redirectHandler = MetadataExport.findRedirectHandler(solrAccess, userProvider, clientProvider, instances, request, pid, null); + if (!pid.contains("_")) { + if (redirectHandler != null && !redirectHandler.isStreamBiblioModsAvaiable()) { + header.setAttribute("status", "deleted"); + // co s tim ?? + } else if (redirectHandler == null){ + header.setAttribute("status", "deleted"); + + } + } + } catch (DOMException | LexerException | IOException | ProxyHandlerException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + } return header; } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java index 2ae8740d14..3fc96a3585 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java @@ -20,6 +20,7 @@ import java.io.StringReader; import java.io.StringWriter; import java.net.URLEncoder; +import java.time.ZonedDateTime; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -170,14 +171,97 @@ public boolean acceptElement(Element element) { return -1; } - public OAIResults findRecords(ProxyFilter proxyFilter, SolrAccess solrAccess,String cursor, String metadataPrefix, int rows) throws IOException, ParserConfigurationException, SAXException { + +// public OAIResults findRecords(SolrAccess solrAccess,String cursor, String metadataPrefix, int configuredMaxRows, String fromParameter, String untilParameter) throws IOException, ParserConfigurationException, SAXException { +// String query = String.format("q=%s&cursorMark=%s&fl=pid+indexed&rows=%d&sort="+getSortField()+"+asc", this.filterQuery, cursor, configuredMaxRows); +// +// if (StringUtils.isAnyString(fromParameter) && StringUtils.isAnyString(untilParameter)) { +// ZonedDateTime fromDate = OAITools.parseISO8601Date(fromParameter); +// ZonedDateTime untilDate = OAITools.parseISO8601Date(untilParameter); +// query = query +String.format("&fq=indexed:[%s+TO+%s]", OAITools.formatForSolr(fromDate), OAITools.formatForSolr(untilDate)); +// } +// if (StringUtils.isAnyString(fromParameter)) { +// ZonedDateTime fromDate = OAITools.parseISO8601Date(fromParameter); +// query = query +String.format("&fq=indexed:[%s+TO+*]", OAITools.formatForSolr(fromDate)); +// } +// +// if (StringUtils.isAnyString(untilParameter)) { +// ZonedDateTime untilDate = OAITools.parseISO8601Date(untilParameter); +// query = query +String.format("&fq=indexed:[*+TO+%s]", OAITools.formatForSolr(untilDate)); +// } +// +// String solrResponseXml = solrAccess.requestWithSelectReturningString(query, "xml"); +// Document document = XMLUtils.parseDocument(new StringReader(solrResponseXml)); +// +// Element result = XMLUtils.findElement(document.getDocumentElement(), new XMLUtils.ElementsFilter() { +// @Override +// public boolean acceptElement(Element element) { +// return element.getNodeName().equals("result"); +// } +// }); +// +// if (result != null) { +// +// String number = result.getAttribute("numFound"); +// +// String solrNextCursor = null; +// Element cursorElement = XMLUtils.findElement(document.getDocumentElement(), (element) -> { +// if (element.getNodeName().equals("str") && element.getAttribute("name").equals("nextCursorMark")) { +// return true; +// } else return false; +// }); +// +// if (cursorElement != null) { +// solrNextCursor = cursorElement.getTextContent(); +// } +// +// List docs = XMLUtils.getElements(result, new XMLUtils.ElementsFilter() { +// @Override +// public boolean acceptElement(Element element) { +// return element.getNodeName().equals("doc"); +// } +// }); +// +// List records = docs.stream().map(doc-> { +// Element pidElm = XMLUtils.findElement(doc, "str"); +// Element dateElm = XMLUtils.findElement(doc, "date"); +// String oaiIdentifier = OAITools.oaiIdentfier(host, pidElm.getTextContent()); +// return new OAIRecord(pidElm.getTextContent(), oaiIdentifier, dateElm != null ? dateElm.getTextContent() : ""); +// }).collect(Collectors.toList()); +// +// +// int rowsInResults = records.size(); +// String resumptionToken = OAITools.generateResumptionToken(metadataPrefix, configuredMaxRows, rowsInResults, solrNextCursor, this.setSpec, fromParameter, untilParameter); +// OAIResults results = new OAIResults(Integer.parseInt(number), resumptionToken, metadataPrefix, records); +// return results; +// } else { +// return null; +// } +// } + + public OAIResults findRecords(ProxyFilter proxyFilter, SolrAccess solrAccess,String cursor, String metadataPrefix, int rows,String fromParameter, String untilParameter) throws IOException, ParserConfigurationException, SAXException { String fq = proxyFilter.newFilter(); - String query = String.format("q=%s&cursorMark=%s&fl=pid+cdk.leaders+cdk.collection+cdk.leader&rows=%d&sort=compositeId+asc", this.filterQuery, cursor, rows); + String query = String.format("q=%s&cursorMark=%s&fl=pid+cdk.collection+cdk.leader+indexed&rows=%d&sort=compositeId+asc", this.filterQuery, cursor, rows); if (fq != null) { String encodedFq = URLEncoder.encode(fq, "UTF-8"); query = query + String.format("&fq=%s", encodedFq); } + if (StringUtils.isAnyString(fromParameter) && StringUtils.isAnyString(untilParameter)) { + ZonedDateTime fromDate = OAITools.parseISO8601Date(fromParameter); + ZonedDateTime untilDate = OAITools.parseISO8601Date(untilParameter); + query = query +String.format("&fq=indexed:[%s+TO+%s]", OAITools.formatForSolr(fromDate), OAITools.formatForSolr(untilDate)); + } + if (StringUtils.isAnyString(fromParameter)) { + ZonedDateTime fromDate = OAITools.parseISO8601Date(fromParameter); + query = query +String.format("&fq=indexed:[%s+TO+*]", OAITools.formatForSolr(fromDate)); + } + + if (StringUtils.isAnyString(untilParameter)) { + ZonedDateTime untilDate = OAITools.parseISO8601Date(untilParameter); + query = query +String.format("&fq=indexed:[*+TO+%s]", OAITools.formatForSolr(untilDate)); + } + String solrResponseXml = solrAccess.requestWithSelectReturningString(query, "xml"); Document document = XMLUtils.parseDocument(new StringReader(solrResponseXml)); @@ -230,10 +314,23 @@ public boolean acceptElement(Element element) { } }); + + Element dateElm = XMLUtils.findElement(doc, new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + String name = element.getAttribute("name"); + return name.equals("indexed"); + + } + }); + + List cdkCollections = collections.stream().map(Element::getTextContent).collect(Collectors.toList()); String oaiIdentifier = OAITools.oaiIdentfier(host, pidElm.getTextContent()); - OAIRecord oaiRecord = new OAIRecord(pidElm.getTextContent(), oaiIdentifier); + OAIRecord oaiRecord = new OAIRecord(pidElm.getTextContent(), oaiIdentifier,dateElm != null ? dateElm.getTextContent() : ""); oaiRecord.setCdkCollections(cdkCollections); + + return oaiRecord; }).collect(Collectors.toList()); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAITools.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAITools.java index 399128c79e..edf6bc51d4 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAITools.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAITools.java @@ -16,10 +16,12 @@ */ package cz.incad.kramerius.rest.oai; +import java.time.LocalDate; import java.time.LocalDateTime; import java.time.OffsetDateTime; import java.time.ZonedDateTime; import java.time.format.DateTimeFormatter; +import java.time.format.DateTimeParseException; import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; @@ -30,6 +32,7 @@ import org.w3c.dom.Document; import org.w3c.dom.Element; +import cz.incad.kramerius.utils.StringUtils; import cz.incad.kramerius.utils.XMLUtils; public class OAITools { @@ -107,6 +110,36 @@ public static String solrCursorMarkFromResumptionToken(String resumptionToken) { } else return null; } + + public static String fromFromResumptionToken(String resumptionToken) { + if (resumptionToken.contains(":")) { + String[] split = resumptionToken.split(":"); + if (split.length > 3) { + String fromUntil=split[3]; + if (fromUntil.contains("from_")) { + int maxIndex = fromUntil.indexOf("_until") > 0 ? fromUntil.indexOf("_until") : fromUntil.length(); + return fromUntil.substring(fromUntil.indexOf("from_")+"from_".length(), maxIndex); + } + return null; + } else return null; + } else return null; + + } + + public static String untilFromResumptionToken(String resumptionToken) { + if (resumptionToken.contains(":")) { + String[] split = resumptionToken.split(":"); + if (split.length > 3) { + String fromUntil=split[3]; + if (fromUntil.contains("until_")) { + return fromUntil.substring(fromUntil.indexOf("until_")+"until_".length(), fromUntil.length()); + } + return null; + } else return null; + } else return null; + + } + /** create OAI identifier */ public static String oaiIdentfier(String host, String pid) { String oaiIdentifier = String.format("oai:%s:%s", host, pid); @@ -129,5 +162,50 @@ public static String pidFromOAIIdentifier(String oaiIdentifier) { return oaiIdentifier.substring(pidIndex); } else return null; } + + + public static ZonedDateTime parseISO8601Date(String dateStr) { + DateTimeFormatter dateFormatter = DateTimeFormatter.ISO_LOCAL_DATE; + DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ISO_DATE_TIME; + try { + // YYYY-MM-DD) + LocalDate date = LocalDate.parse(dateStr, dateFormatter); + return date.atStartOfDay(ZonedDateTime.now().getZone()); + } catch (DateTimeParseException e) { + // YYYY-MM-DDThh:mm:ssZ + try { + LocalDateTime dateTime = LocalDateTime.parse(dateStr, dateTimeFormatter); + return dateTime.atZone(ZonedDateTime.now().getZone()); + } catch (DateTimeParseException ex) { + // ZonedDateTime (full UTC) + return ZonedDateTime.parse(dateStr, dateTimeFormatter); + } + } + } + public static String formatForSolr(ZonedDateTime dateTime) { + DateTimeFormatter solrDateFormat = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss'Z'"); + return dateTime.withZoneSameInstant(dateTime.getZone()).format(solrDateFormat); + } + + public static String generateResumptionToken(String metadataPrefix, int rowsInPage, int configuredRowsInPage, String solrNextCursor, String setSpec, String from, String until) { + if (configuredRowsInPage == rowsInPage && solrNextCursor != null) { + StringBuilder builder = new StringBuilder().append(solrNextCursor).append(":").append(setSpec).append(":").append(metadataPrefix); + if (StringUtils.isAnyString(from) || StringUtils.isAnyString(until)) { + builder.append(":"); + if (StringUtils.isAnyString(from)) { + builder.append("from_").append(from); + + if (StringUtils.isAnyString(until)) { + builder.append("_"); + } + } + if (StringUtils.isAnyString(until)) { + builder.append("until_").append(until); + } + } + return builder.toString(); + } else return null; + } + } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java index 2139ebb715..0630e1da52 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java @@ -19,6 +19,7 @@ import java.io.IOException; import java.net.MalformedURLException; import java.net.URL; +import java.time.DateTimeException; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -42,6 +43,7 @@ import cz.incad.kramerius.rest.oai.exceptions.OAIException; import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.ApplicationURL; +import cz.incad.kramerius.utils.StringUtils; import cz.incad.kramerius.utils.XMLUtils; import cz.incad.kramerius.utils.conf.KConfiguration; @@ -220,10 +222,14 @@ public void perform(Provider userProvider,Provider clientProvider, String baseUrl = ApplicationURL.applicationURL(request); URL urlObject = new URL(baseUrl); OAISets sets = new OAISets(configManager, urlObject.getHost()); - String set = request.getParameter("set"); - String resumptionToken = request.getParameter("resumptionToken"); - String metadataPrefix = request.getParameter("metadataPrefix"); + String set = request.getParameter(SET_PARAMETER); + String resumptionToken = request.getParameter(RESUMPTION_TOKEN_PARAMETER); + String metadataPrefix = request.getParameter(METADATA_PREFIX_PARAMETER); + String from = request.getParameter(FROM_PARAMETER); + String until = request.getParameter(UNTIL_PARAMETER); + + if (metadataPrefix != null || resumptionToken != null) { int rows = KConfiguration.getInstance().getConfiguration().getInt(REPOSITORY_ROWS_IN_RESULTS,600); if (set != null) { @@ -240,6 +246,22 @@ public void perform(Provider userProvider,Provider clientProvider, throw new OAIException(ErrorCode.badResumptionToken, OAIVerb.ListRecords, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); } } + + if (StringUtils.isAnyString(from)) { + try { + OAITools.parseISO8601Date(from); + } catch (DateTimeException e) { + throw new OAIException(ErrorCode.badArgument, OAIVerb.ListRecords, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata, "illegal value of from"); + } + } + if (StringUtils.isAnyString(until)) { + try { + OAITools.parseISO8601Date(until); + } catch (DateTimeException e) { + throw new OAIException(ErrorCode.badArgument, OAIVerb.ListRecords, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata,"illegal value of until"); + } + } + selectedMetadata = MetadataExport.findByPrefix(metadataPrefix); @@ -256,20 +278,18 @@ public void perform(Provider userProvider,Provider clientProvider, if (resumptionToken != null) { String solrCursor = OAITools.solrCursorMarkFromResumptionToken(resumptionToken); - results = selectedSet.findRecords(proxyFilter, solrAccess, solrCursor,metadataPrefix,rows); + results = selectedSet.findRecords(proxyFilter, solrAccess, solrCursor,metadataPrefix,rows, from, until); for (OAIRecord oaiRec : results.getRecords()) { Element record= doc.createElement("record"); - Element header = oaiRec.toHeader(doc, selectedSet.getSetSpec()); - - // Instances instances, HttpServletRequest request, Document owningDocument, String oaiIdentifier, MetadataExport export, OAISet set + Element header = oaiRec.toHeader(doc, selectedSet, solrAccess, userProvider, clientProvider, instances, request, null); Element metadata = doc.createElement("metadata"); Element metadataElm = oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,selectedSet); if (metadataElm != null) { metadata.appendChild(metadataElm); } else { - record.setAttribute("deleted","true"); + header.setAttribute("status","deleted"); } record.appendChild(header); @@ -280,18 +300,19 @@ public void perform(Provider userProvider,Provider clientProvider, } } else { - results = selectedSet.findRecords(proxyFilter, solrAccess,"*", metadataPrefix,rows); + results = selectedSet.findRecords(proxyFilter, solrAccess,"*", metadataPrefix,rows, from, until); for (OAIRecord oaiRec : results.getRecords()) { Element record= doc.createElement("record"); - Element header = oaiRec.toHeader(doc, selectedSet.getSetSpec()); + Element header = oaiRec.toHeader(doc, selectedSet, solrAccess, userProvider, clientProvider, instances, request, null); + //Element header = oaiRec.toHeader(doc, selectedSet); Element metadata = doc.createElement("metadata"); Element metadataElm = oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,selectedSet); if (metadataElm != null) { metadata.appendChild(metadataElm); } else { - record.setAttribute("deleted","true"); + header.setAttribute("status","deleted"); } record.appendChild(header); @@ -335,9 +356,14 @@ public void perform(Provider userProvider,Provider clientProvider, URL urlObject = new URL(baseUrl); OAISets sets = new OAISets(configManager, urlObject.getHost()); - String set = request.getParameter("set"); - String resumptionToken = request.getParameter("resumptionToken"); - String metadataPrefix = request.getParameter("metadataPrefix"); + + String set = request.getParameter(SET_PARAMETER); + String resumptionToken = request.getParameter(RESUMPTION_TOKEN_PARAMETER); + String metadataPrefix = request.getParameter(METADATA_PREFIX_PARAMETER); + String from = request.getParameter(FROM_PARAMETER); + String until = request.getParameter(UNTIL_PARAMETER); + + if (metadataPrefix != null || resumptionToken != null) { int rows = KConfiguration.getInstance().getConfiguration().getInt(REPOSITORY_ROWS_IN_RESULTS,600); @@ -351,6 +377,21 @@ public void perform(Provider userProvider,Provider clientProvider, } } + if (StringUtils.isAnyString(from)) { + try { + OAITools.parseISO8601Date(from); + } catch (DateTimeException e) { + throw new OAIException(ErrorCode.badArgument, OAIVerb.ListRecords, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata, "illegal value of from"); + } + } + if (StringUtils.isAnyString(until)) { + try { + OAITools.parseISO8601Date(until); + } catch (DateTimeException e) { + throw new OAIException(ErrorCode.badArgument, OAIVerb.ListRecords, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata,"illegal value of until"); + } + } + selectedMetadata = MetadataExport.findByPrefix(metadataPrefix); if (selectedMetadata != null) { @@ -365,11 +406,17 @@ public void perform(Provider userProvider,Provider clientProvider, OAIResults results = null; if (resumptionToken != null) { String solrCursor = OAITools.solrCursorMarkFromResumptionToken(resumptionToken); - results = selectedSet.findRecords(proxyFilter, solrAccess, solrCursor,metadataPrefix,rows); - for (OAIRecord oaiRec : results.getRecords()) { identify.appendChild(oaiRec.toHeader(doc, selectedSet.getSetSpec()));} + results = selectedSet.findRecords(proxyFilter, solrAccess, solrCursor,metadataPrefix,rows, from, until); + for (OAIRecord oaiRec : results.getRecords()) { + Element header = oaiRec.toHeader(doc, selectedSet, solrAccess, userProvider, clientProvider, instances, request, null); + identify.appendChild(header);} } else { - results = selectedSet.findRecords(proxyFilter, solrAccess,"*", metadataPrefix,rows); - for (OAIRecord oaiRec : results.getRecords()) { identify.appendChild(oaiRec.toHeader(doc, selectedSet.getSetSpec()));} + results = selectedSet.findRecords(proxyFilter, solrAccess,"*", metadataPrefix,rows, from, until); + for (OAIRecord oaiRec : results.getRecords()) { + Element header = oaiRec.toHeader(doc, selectedSet, solrAccess, userProvider, clientProvider, instances, request, null); + identify.appendChild(header); + + } } if (results.getResumptionToken()!= null) { @@ -420,8 +467,8 @@ public void perform(Provider userProvider,Provider clientProvider, if (oaiRec != null) { Element record= doc.createElement("record"); - Element header = oaiRec.toHeader(doc, null); - + + Element header = oaiRec.toHeader(doc, null, solrAccess, userProvider, clientProvider, instances, request, null); Element metadata = doc.createElement("metadata"); Element metadataElm = oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,null); @@ -452,6 +499,12 @@ public void perform(Provider userProvider,Provider clientProvider, } }; + private static final String METADATA_PREFIX_PARAMETER = "metadataPrefix"; + private static final String RESUMPTION_TOKEN_PARAMETER = "resumptionToken"; + private static final String SET_PARAMETER = "set"; + private static final String UNTIL_PARAMETER = "until"; + private static final String FROM_PARAMETER = "from"; + private static final String REPOSITORY_ADMIN_EMAIL = "oai.adminEmail"; private static final String REPOSITORY_BASE_URL = "oai.baseUrl"; private static final String REPOSITORY_NAME = "oai.repositoryName"; diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/exceptions/OAIException.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/exceptions/OAIException.java index 110cd8a8e0..32416ef2da 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/exceptions/OAIException.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/exceptions/OAIException.java @@ -34,6 +34,7 @@ import cz.incad.kramerius.rest.oai.MetadataExport; import cz.incad.kramerius.rest.oai.OAISet; import cz.incad.kramerius.rest.oai.OAIVerb; +import cz.incad.kramerius.utils.StringUtils; import cz.incad.kramerius.utils.XMLUtils; import static cz.incad.kramerius.rest.oai.OAITools.*; @@ -61,18 +62,25 @@ protected OAIException(Response response) { public OAIException(ErrorCode oaiErrorCode,OAIVerb verb, OAISet set,String baseUrl,MetadataExport metadata) { this(Response.status(Response.Status.BAD_REQUEST) .type(MediaType.APPLICATION_XML) - .entity(buildXml(oaiErrorCode, verb, set, baseUrl, metadata)) + .entity(buildXml(oaiErrorCode, verb, set, baseUrl, metadata, null)) + .build()); + } + + public OAIException(ErrorCode oaiErrorCode,OAIVerb verb, OAISet set,String baseUrl,MetadataExport metadata, String message) { + this(Response.status(Response.Status.BAD_REQUEST) + .type(MediaType.APPLICATION_XML) + .entity(buildXml(oaiErrorCode, verb, set, baseUrl, metadata, message)) .build()); } public OAIException(int errorCode, ErrorCode oaiErrorCode,OAIVerb verb, OAISet set,String baseUrl,MetadataExport metadata) { this(Response.status(errorCode) .type(MediaType.APPLICATION_XML) - .entity(buildXml(oaiErrorCode, verb, set, baseUrl, metadata)) + .entity(buildXml(oaiErrorCode, verb, set, baseUrl, metadata, null)) .build()); } - private static String buildXml(ErrorCode oaiErrorCode, OAIVerb verb, OAISet set,String baseUrl,MetadataExport metadata) { + private static String buildXml(ErrorCode oaiErrorCode, OAIVerb verb, OAISet set,String baseUrl,MetadataExport metadata,String message) { try { Document oai = createOAIDocument(); Element oaiRoot = oai.getDocumentElement(); @@ -82,6 +90,8 @@ private static String buildXml(ErrorCode oaiErrorCode, OAIVerb verb, OAISet set, error.setAttribute("code", oaiErrorCode.name()); oaiRoot.appendChild(error); + if (StringUtils.isAnyString(message)) { error.setTextContent(message); } + StringWriter writer = new StringWriter(); XMLUtils.print(oai, writer); return writer.toString(); From 97542f5faa9a13bc87a82475c714af804d67231c Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 2 Sep 2024 11:04:20 +0200 Subject: [PATCH 204/235] Issue https://github.com/ceskaexpedice/ceska-digitalni-knihovna/issues/76 --- .../cz/incad/kramerius/solr/SolrKeys.java | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 shared/common/src/main/java/cz/incad/kramerius/solr/SolrKeys.java diff --git a/shared/common/src/main/java/cz/incad/kramerius/solr/SolrKeys.java b/shared/common/src/main/java/cz/incad/kramerius/solr/SolrKeys.java new file mode 100644 index 0000000000..7235e7f225 --- /dev/null +++ b/shared/common/src/main/java/cz/incad/kramerius/solr/SolrKeys.java @@ -0,0 +1,34 @@ +/* + * Copyright (C) Aug 1, 2024 Pavel Stastny + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +package cz.incad.kramerius.solr; + +public class SolrKeys { + + /** Composite id */ + public static final String SOLR_SEARCH_USE_COMPOSITE_ID = "solrSearch.useCompositeId"; + + /** Max value for hl.fragsize parameter */ + public static final int MAX_HL_FRAGSIZE = 120; + /** Max value for hl.snippets parameter */ + public static final int MAX_HL_SNIPPETS = 10; + /** Max value for combination of the parameters hl.fragsize*hl.snippets */ + public static final int MAX_HL_COMBINATION = 300; + + + + private SolrKeys( ) {} +} From e0d620195efba40d972bdfa94c3932a25ca1d999 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 2 Sep 2024 11:48:42 +0200 Subject: [PATCH 205/235] Issue https://github.com/ceskaexpedice/ceska-digitalni-knihovna/issues/76 --- .../cz/incad/kramerius/rest/oai/OAIVerb.java | 104 +++++++++++------- 1 file changed, 62 insertions(+), 42 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java index 0630e1da52..103a2195e0 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAIVerb.java @@ -279,47 +279,58 @@ public void perform(Provider userProvider,Provider clientProvider, if (resumptionToken != null) { String solrCursor = OAITools.solrCursorMarkFromResumptionToken(resumptionToken); results = selectedSet.findRecords(proxyFilter, solrAccess, solrCursor,metadataPrefix,rows, from, until); - for (OAIRecord oaiRec : results.getRecords()) { - - Element record= doc.createElement("record"); - Element header = oaiRec.toHeader(doc, selectedSet, solrAccess, userProvider, clientProvider, instances, request, null); - - Element metadata = doc.createElement("metadata"); - Element metadataElm = oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,selectedSet); - if (metadataElm != null) { - metadata.appendChild(metadataElm); - } else { - header.setAttribute("status","deleted"); + + if (results.getCompleteListSize() > 0) { + for (OAIRecord oaiRec : results.getRecords()) { + + Element record= doc.createElement("record"); + Element header = oaiRec.toHeader(doc, selectedSet, solrAccess, userProvider, clientProvider, instances, request, null); + + Element metadata = doc.createElement("metadata"); + Element metadataElm = oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,selectedSet); + if (metadataElm != null) { + metadata.appendChild(metadataElm); + } else { + header.setAttribute("status","deleted"); + } + + record.appendChild(header); + record.appendChild(metadata); + + identify.appendChild(record); + + } - - record.appendChild(header); - record.appendChild(metadata); - - identify.appendChild(record); - - + } else { + throw new OAIException(ErrorCode.noRecordsMatch, OAIVerb.ListIdentifiers, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); } + } else { results = selectedSet.findRecords(proxyFilter, solrAccess,"*", metadataPrefix,rows, from, until); - for (OAIRecord oaiRec : results.getRecords()) { - - Element record= doc.createElement("record"); - Element header = oaiRec.toHeader(doc, selectedSet, solrAccess, userProvider, clientProvider, instances, request, null); - //Element header = oaiRec.toHeader(doc, selectedSet); - - Element metadata = doc.createElement("metadata"); - Element metadataElm = oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,selectedSet); - if (metadataElm != null) { - metadata.appendChild(metadataElm); - } else { - header.setAttribute("status","deleted"); + if (results.getCompleteListSize() > 0) { + for (OAIRecord oaiRec : results.getRecords()) { + + Element record= doc.createElement("record"); + Element header = oaiRec.toHeader(doc, selectedSet, solrAccess, userProvider, clientProvider, instances, request, null); + //Element header = oaiRec.toHeader(doc, selectedSet); + + Element metadata = doc.createElement("metadata"); + Element metadataElm = oaiRec.toMetadata(solrAccess, userProvider, clientProvider, instances, request, doc, oaiRec.getIdentifier(), selectedMetadata,selectedSet); + if (metadataElm != null) { + metadata.appendChild(metadataElm); + } else { + header.setAttribute("status","deleted"); + } + + record.appendChild(header); + record.appendChild(metadata); + + identify.appendChild(record); } - - record.appendChild(header); - record.appendChild(metadata); - - identify.appendChild(record); + } else { + throw new OAIException(ErrorCode.noRecordsMatch, OAIVerb.ListIdentifiers, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); } + } if (results.getResumptionToken()!= null) { @@ -407,15 +418,24 @@ public void perform(Provider userProvider,Provider clientProvider, if (resumptionToken != null) { String solrCursor = OAITools.solrCursorMarkFromResumptionToken(resumptionToken); results = selectedSet.findRecords(proxyFilter, solrAccess, solrCursor,metadataPrefix,rows, from, until); - for (OAIRecord oaiRec : results.getRecords()) { - Element header = oaiRec.toHeader(doc, selectedSet, solrAccess, userProvider, clientProvider, instances, request, null); - identify.appendChild(header);} + if (results.getCompleteListSize() > 0) { + for (OAIRecord oaiRec : results.getRecords()) { + Element header = oaiRec.toHeader(doc, selectedSet, solrAccess, userProvider, clientProvider, instances, request, null); + identify.appendChild(header);} + } else { + throw new OAIException(ErrorCode.noRecordsMatch, OAIVerb.ListIdentifiers, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); + } + } else { results = selectedSet.findRecords(proxyFilter, solrAccess,"*", metadataPrefix,rows, from, until); - for (OAIRecord oaiRec : results.getRecords()) { - Element header = oaiRec.toHeader(doc, selectedSet, solrAccess, userProvider, clientProvider, instances, request, null); - identify.appendChild(header); - + if (results.getCompleteListSize() > 0) { + for (OAIRecord oaiRec : results.getRecords()) { + Element header = oaiRec.toHeader(doc, selectedSet, solrAccess, userProvider, clientProvider, instances, request, null); + identify.appendChild(header); + + } + } else { + throw new OAIException(ErrorCode.noRecordsMatch, OAIVerb.ListIdentifiers, selectedSet, ApplicationURL.applicationURL(request),selectedMetadata); } } From 0a0ded49db665d3100bd69c60eadd43b01be1cff Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Mon, 2 Sep 2024 15:21:17 +0200 Subject: [PATCH 206/235] Users resource; handle exceptions --- .../user/V5ForwardUserHandler.java | 20 +++++++++++++------ .../user/V7ForwardUserHandler.java | 20 +++++++++++++------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java index 4387054198..83d62ab95b 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V5ForwardUserHandler.java @@ -4,6 +4,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; import javax.ws.rs.core.Response; @@ -13,7 +14,9 @@ import org.json.JSONObject; import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientHandlerException; import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.UniformInterfaceException; import cz.incad.kramerius.SolrAccess; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; @@ -40,12 +43,17 @@ protected String forwardUrl() { @Override public Pair> user() throws ProxyHandlerException { - String baseurl = forwardUrl(); - String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/user"; - ClientResponse fResponse = super.forwardedResponse(url); - String entity = fResponse.getEntity(String.class); - JSONObject jObject = new JSONObject(entity); - return userFromJSON(jObject); + try { + String baseurl = forwardUrl(); + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/user"; + ClientResponse fResponse = super.forwardedResponse(url); + String entity = fResponse.getEntity(String.class); + JSONObject jObject = new JSONObject(entity); + return userFromJSON(jObject); + } catch (ClientHandlerException | UniformInterfaceException | JSONException | ProxyHandlerException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + return null; + } } public static Pair> userFromJSON(JSONObject json) throws JSONException { diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java index 1780f76727..eaaf2e81ef 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/user/V7ForwardUserHandler.java @@ -4,6 +4,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.logging.Level; import org.apache.commons.lang3.tuple.Pair; import org.json.JSONArray; @@ -11,7 +12,9 @@ import org.json.JSONObject; import com.sun.jersey.api.client.Client; +import com.sun.jersey.api.client.ClientHandlerException; import com.sun.jersey.api.client.ClientResponse; +import com.sun.jersey.api.client.UniformInterfaceException; import cz.incad.kramerius.SolrAccess; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; @@ -45,12 +48,17 @@ protected String forwardUrl() { @Override public Pair> user() throws ProxyHandlerException { - String baseurl = forwardUrl(); - String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/user"; - ClientResponse fResponse = super.forwardedResponse(url); - String entity = fResponse.getEntity(String.class); - JSONObject jObject = new JSONObject(entity); - return userFromJSON(jObject); + try { + String baseurl = forwardUrl(); + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/user"; + ClientResponse fResponse = super.forwardedResponse(url); + String entity = fResponse.getEntity(String.class); + JSONObject jObject = new JSONObject(entity); + return userFromJSON(jObject); + } catch (ClientHandlerException | UniformInterfaceException | JSONException | ProxyHandlerException e) { + LOGGER.log(Level.SEVERE,e.getMessage(),e); + return null; + } } public static Pair> userFromJSON(JSONObject json) throws JSONException { From ce8f5219b974788097998949dadcdb47d45affb8 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 4 Sep 2024 06:12:07 +0200 Subject: [PATCH 207/235] Issue https://github.com/ceskaexpedice/ceska-digitalni-knihovna/issues/76 --- rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java index 3fc96a3585..1d96178555 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/oai/OAISet.java @@ -251,13 +251,10 @@ public OAIResults findRecords(ProxyFilter proxyFilter, SolrAccess solrAccess,Str ZonedDateTime fromDate = OAITools.parseISO8601Date(fromParameter); ZonedDateTime untilDate = OAITools.parseISO8601Date(untilParameter); query = query +String.format("&fq=indexed:[%s+TO+%s]", OAITools.formatForSolr(fromDate), OAITools.formatForSolr(untilDate)); - } - if (StringUtils.isAnyString(fromParameter)) { + } else if (StringUtils.isAnyString(fromParameter)) { ZonedDateTime fromDate = OAITools.parseISO8601Date(fromParameter); query = query +String.format("&fq=indexed:[%s+TO+*]", OAITools.formatForSolr(fromDate)); - } - - if (StringUtils.isAnyString(untilParameter)) { + } else if (StringUtils.isAnyString(untilParameter)) { ZonedDateTime untilDate = OAITools.parseISO8601Date(untilParameter); query = query +String.format("&fq=indexed:[*+TO+%s]", OAITools.formatForSolr(untilDate)); } From fdf17e1fb0de02250571324a4ede3074ffaebd29 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 20 Sep 2024 05:39:40 +0200 Subject: [PATCH 208/235] Debug messages --- .../apiNew/client/v60/redirection/ProxyHandlerSupport.java | 1 + .../apiNew/client/v60/redirection/item/V7ForwardHandler.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index d6ceebca0d..78f8793887 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -124,6 +124,7 @@ public Response buildForwardResponseGET(String url, String mimetype, String pid, throws ProxyHandlerException { WebResource.Builder b = buidFowrardResponse(url); ClientResponse response = b.get(ClientResponse.class); + LOGGER.info("Status code response "+response.getStatus()); if (response.getStatus() == 200) { String responseMimeType = response.getType().toString(); InputStream is = response.getEntityInputStream(); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index 75235a0fc9..2b502f03fb 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -169,10 +169,11 @@ public Response textOCR(RequestMethodName method) throws ProxyHandlerException { public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { String baseurl = super.baseUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/item/" + this.pid - + "/streams/TEXT_OCR"; + + "/streams/ALTO"; if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { + LOGGER.info(String.format("ALTO OCR url, pid, deleteTrigger = %s,%s, %b", (url != null ? url : "null"), "",false)); return buildForwardResponseGET(url, null,false); } } From cb2b97ea70921486927e982f0ad506a90a5dd8f6 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 20 Sep 2024 05:48:33 +0200 Subject: [PATCH 209/235] Fix alto stream --- .../apiNew/client/v60/redirection/item/V7ForwardHandler.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index 2b502f03fb..ab45da086c 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -167,13 +167,12 @@ public Response textOCR(RequestMethodName method) throws ProxyHandlerException { @Override public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { - String baseurl = super.baseUrl(); + String baseurl = this.forwardUrl(); String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/cdk/v7.0/forward/item/" + this.pid + "/streams/ALTO"; if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - LOGGER.info(String.format("ALTO OCR url, pid, deleteTrigger = %s,%s, %b", (url != null ? url : "null"), "",false)); return buildForwardResponseGET(url, null,false); } } From 38a2057272050d3a1cc7be93aaf43f4dd667cb86 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 20 Sep 2024 06:11:03 +0200 Subject: [PATCH 210/235] Fix alto - mimetype --- .../apiNew/client/v60/redirection/item/V5ForwardHandler.java | 2 +- .../apiNew/client/v60/redirection/item/V7ForwardHandler.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java index 61714dd32f..df0721a777 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java @@ -134,7 +134,7 @@ public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, null, false); + return buildForwardResponseGET(url, "application/xml;charset=utf-8", false); } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index ab45da086c..059972b0d7 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -173,7 +173,7 @@ public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, null,false); + return buildForwardResponseGET(url, "application/xml;charset=utf-8",false); } } From 3a00998e943111dc86b0863ea90d5634d17811f0 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 20 Sep 2024 06:26:39 +0200 Subject: [PATCH 211/235] Debug message - mimetype & status code --- .../rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 78f8793887..7e07ad7521 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -124,7 +124,7 @@ public Response buildForwardResponseGET(String url, String mimetype, String pid, throws ProxyHandlerException { WebResource.Builder b = buidFowrardResponse(url); ClientResponse response = b.get(ClientResponse.class); - LOGGER.info("Status code response "+response.getStatus()); + LOGGER.info("Status code response "+response.getStatus() + ",Mimetype "+mimetype); if (response.getStatus() == 200) { String responseMimeType = response.getType().toString(); InputStream is = response.getEntityInputStream(); From 686926fd37c5b355bf2ca33dc388e78eef0fa257 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 20 Sep 2024 06:39:58 +0200 Subject: [PATCH 212/235] Pid and mimetype --- .../apiNew/client/v60/redirection/item/V5ForwardHandler.java | 2 +- .../apiNew/client/v60/redirection/item/V7ForwardHandler.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java index df0721a777..fb0693adfd 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java @@ -134,7 +134,7 @@ public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, "application/xml;charset=utf-8", false); + return buildForwardResponseGET(url, "application/xml;charset=utf-8", null, false); } } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index 059972b0d7..5173ebffe5 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -173,7 +173,7 @@ public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - return buildForwardResponseGET(url, "application/xml;charset=utf-8",false); + return buildForwardResponseGET(url, "application/xml;charset=utf-8",null,false); } } From 91783bac209a6ed6d06f91a07ca11de493b69b75 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 20 Sep 2024 06:48:24 +0200 Subject: [PATCH 213/235] Forward handler --- .../rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java | 2 +- .../apiNew/client/v60/redirection/item/V7ForwardHandler.java | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 7e07ad7521..8f4e36b804 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -124,7 +124,7 @@ public Response buildForwardResponseGET(String url, String mimetype, String pid, throws ProxyHandlerException { WebResource.Builder b = buidFowrardResponse(url); ClientResponse response = b.get(ClientResponse.class); - LOGGER.info("Status code response "+response.getStatus() + ",Mimetype "+mimetype); + LOGGER.info("Status code response "+response.getStatus() + ",Mimetype "+mimetype+", pid "+pid+", deleteTrigger "+deleteTrigger); if (response.getStatus() == 200) { String responseMimeType = response.getType().toString(); InputStream is = response.getEntityInputStream(); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index 5173ebffe5..bea9effa9d 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -173,6 +173,7 @@ public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { + LOGGER.info("buildForwardResponseGET( url = "+url+", mimetype=\"application/xml;charset=utf-8\", pid=null, deleteTrigger=false)"); return buildForwardResponseGET(url, "application/xml;charset=utf-8",null,false); } } From 75f3a502113ee711acde1c57671ba81c20503f24 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 20 Sep 2024 06:56:32 +0200 Subject: [PATCH 214/235] Debug messages disabled --- .../rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java | 2 +- .../apiNew/client/v60/redirection/item/V7ForwardHandler.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index 8f4e36b804..bd9e0c109c 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -124,7 +124,7 @@ public Response buildForwardResponseGET(String url, String mimetype, String pid, throws ProxyHandlerException { WebResource.Builder b = buidFowrardResponse(url); ClientResponse response = b.get(ClientResponse.class); - LOGGER.info("Status code response "+response.getStatus() + ",Mimetype "+mimetype+", pid "+pid+", deleteTrigger "+deleteTrigger); + //LOGGER.info("Status code response "+response.getStatus() + ",Mimetype "+mimetype+", pid "+pid+", deleteTrigger "+deleteTrigger); if (response.getStatus() == 200) { String responseMimeType = response.getType().toString(); InputStream is = response.getEntityInputStream(); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java index bea9effa9d..d98097f01a 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V7ForwardHandler.java @@ -173,7 +173,7 @@ public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { if (method == RequestMethodName.head) { return buildForwardResponseHEAD(url); } else { - LOGGER.info("buildForwardResponseGET( url = "+url+", mimetype=\"application/xml;charset=utf-8\", pid=null, deleteTrigger=false)"); + //LOGGER.info("buildForwardResponseGET( url = "+url+", mimetype=\"application/xml;charset=utf-8\", pid=null, deleteTrigger=false)"); return buildForwardResponseGET(url, "application/xml;charset=utf-8",null,false); } } From a00caf9f86a55bb2a2524dd9602f38fc86c86b71 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Fri, 20 Sep 2024 15:43:31 +0200 Subject: [PATCH 215/235] V5Forward handler --- .../apiNew/client/v60/redirection/item/V5ForwardHandler.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java index fb0693adfd..e58cf3462e 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/item/V5ForwardHandler.java @@ -115,7 +115,7 @@ public Response zoomifyTile(String tileGroupStr, String tileStr) throws ProxyHan @Override public Response textOCR(RequestMethodName method) throws ProxyHandlerException { String baseurl = this.forwardUrl(); - String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/" + this.pid + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/item/" + this.pid + "/streams/TEXT_OCR"; if (method == RequestMethodName.head) { @@ -127,8 +127,9 @@ public Response textOCR(RequestMethodName method) throws ProxyHandlerException { @Override public Response altoOCR(RequestMethodName method) throws ProxyHandlerException { + String baseurl = this.forwardUrl(); - String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/" + this.pid + String url = baseurl + (baseurl.endsWith("/") ? "" : "/") + "api/v5.0/cdk/forward/item/" + this.pid + "/streams/ALTO"; if (method == RequestMethodName.head) { From 36c7de42ba89031323567bffd021db6727a2caa7 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sat, 21 Sep 2024 23:46:58 +0200 Subject: [PATCH 216/235] channels - pid solr introspect --- .../rest/apiNew/admin/v10/ItemsResource.java | 43 ++++++++++++++++++- .../cz/incad/kramerius/cdk/ChannelUtils.java | 21 +++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java index 32f7e0dd35..06bf41f301 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java @@ -1,19 +1,28 @@ package cz.incad.kramerius.rest.apiNew.admin.v10; +import cz.incad.kramerius.cdk.ChannelUtils; import cz.incad.kramerius.repository.RepositoryApi; import cz.incad.kramerius.repository.utils.Utils; +import cz.incad.kramerius.rest.api.k5.client.utils.UsersUtils; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance.InstanceType; import cz.incad.kramerius.rest.apiNew.exceptions.BadRequestException; import cz.incad.kramerius.rest.apiNew.exceptions.ForbiddenException; import cz.incad.kramerius.rest.apiNew.exceptions.InternalErrorException; import cz.incad.kramerius.security.Role; import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.StringUtils; +import cz.incad.kramerius.utils.conf.KConfiguration; import cz.incad.kramerius.utils.java.Pair; import org.apache.commons.io.IOUtils; import org.dom4j.Document; import org.json.JSONArray; import org.json.JSONObject; +import com.google.inject.Inject; +import com.sun.jersey.api.client.Client; + import javax.inject.Provider; import javax.ws.rs.*; import javax.ws.rs.core.MediaType; @@ -21,6 +30,7 @@ import javax.ws.rs.core.StreamingOutput; import java.io.InputStream; import java.util.Arrays; +import java.util.Iterator; import java.util.List; import java.util.logging.Level; import java.util.logging.Logger; @@ -45,6 +55,16 @@ public class ItemsResource extends AdminApiResource { @javax.inject.Inject Provider userProvider; + @Inject + private Instances libraries; + + private Client client; + + + public ItemsResource() { + super(); + this.client = Client.create(); + } /** * Returns array of pids (with titles) that have given model. Only partial array with offset & limit. @@ -163,7 +183,28 @@ public Response checkItemExists(@PathParam("pid") String pid) { throw new InternalErrorException(e.getMessage()); } } - + + @GET + @Path("{pid}/solr/instintrospect") + @Produces(MediaType.APPLICATION_JSON) + public Response introspectPidInInstances(@PathParam("pid") String pid) { + JSONObject obj = new JSONObject(); + List instances = libraries.enabledInstances(); + for(OneInstance inst:instances) { + String library = inst.getName(); + boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + library + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + library + ".licenses") : false; + String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".forwardurl"); + String solrChannelUrl = ChannelUtils.solrChannelUrl(inst.getInstanceType().name(), channel); + InstanceType instType = inst.getInstanceType(); + String solrPid = ChannelUtils.solrChannelPid(this.client, channel, solrChannelUrl, instType.name(), pid); + if (solrPid != null) { + obj.put(library, new JSONObject(solrPid)); + } + } + return Response.ok(obj.toString()).build(); + } + + @GET @Path("{pid}/foxml") @Produces(MediaType.APPLICATION_XML) diff --git a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java index 8fd2511812..be9ba1e73b 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java @@ -109,6 +109,27 @@ public static void checkSolrChannelEndpoint(Client client, String ac,String full } else throw new IllegalStateException(String.format("Channel for %s(%s) doesnt work ", ac, fullChannelUrl)); } + public static String solrChannelPid(Client client, String ac, String fullChannelUrl, String apiVersion, String pid) { + if (apiVersion.toLowerCase().equals("v5")) { + WebResource configResource = client.resource(fullChannelUrl+"/select?q=PID:\""+pid+"\"&rows=0&wt=json"); + ClientResponse solrResource = configResource.accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + if (solrResource.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + String entity = solrResource.getEntity(String.class); + return entity; + } + } else { + + WebResource configResource = client.resource(fullChannelUrl+"/select?q=pid:\""+pid+"\"&rows=0&wt=json"); + ClientResponse solrResource = configResource.accept(MediaType.APPLICATION_JSON) + .get(ClientResponse.class); + if (solrResource.getStatus() == ClientResponse.Status.OK.getStatusCode()) { + String entity = solrResource.getEntity(String.class); + return entity; + } + } + return null; + } } From b60c540510e1d6a9bf102ee5f0014a5252180083 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 22 Sep 2024 00:15:24 +0200 Subject: [PATCH 217/235] Introspect -> boolean flag --- .../rest/apiNew/admin/v10/ItemsResource.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java index 06bf41f301..3ac12bb9bb 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java @@ -193,12 +193,14 @@ public Response introspectPidInInstances(@PathParam("pid") String pid) { for(OneInstance inst:instances) { String library = inst.getName(); boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + library + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + library + ".licenses") : false; - String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".forwardurl"); - String solrChannelUrl = ChannelUtils.solrChannelUrl(inst.getInstanceType().name(), channel); - InstanceType instType = inst.getInstanceType(); - String solrPid = ChannelUtils.solrChannelPid(this.client, channel, solrChannelUrl, instType.name(), pid); - if (solrPid != null) { - obj.put(library, new JSONObject(solrPid)); + if(channelAccess) { + String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".forwardurl"); + String solrChannelUrl = ChannelUtils.solrChannelUrl(inst.getInstanceType().name(), channel); + InstanceType instType = inst.getInstanceType(); + String solrPid = ChannelUtils.solrChannelPid(this.client, channel, solrChannelUrl, instType.name(), pid); + if (solrPid != null) { + obj.put(library, new JSONObject(solrPid)); + } } } return Response.ok(obj.toString()).build(); From d80e6a8ca15806d8ce9101945238032bdc7536d6 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 22 Sep 2024 00:34:45 +0200 Subject: [PATCH 218/235] Qeuery encoding --- .../java/cz/incad/kramerius/cdk/ChannelUtils.java | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java index be9ba1e73b..07d91cca79 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java @@ -1,5 +1,7 @@ package cz.incad.kramerius.cdk; +import java.io.UnsupportedEncodingException; +import java.net.URLEncoder; import java.util.Map; import java.util.logging.Logger; @@ -109,9 +111,10 @@ public static void checkSolrChannelEndpoint(Client client, String ac,String full } else throw new IllegalStateException(String.format("Channel for %s(%s) doesnt work ", ac, fullChannelUrl)); } - public static String solrChannelPid(Client client, String ac, String fullChannelUrl, String apiVersion, String pid) { + public static String solrChannelPid(Client client, String ac, String fullChannelUrl, String apiVersion, String pid) throws UnsupportedEncodingException { if (apiVersion.toLowerCase().equals("v5")) { - WebResource configResource = client.resource(fullChannelUrl+"/select?q=PID:\""+pid+"\"&rows=0&wt=json"); + String query = URLEncoder.encode( "PID:\""+pid+"\"", "UTF-8"); + WebResource configResource = client.resource(fullChannelUrl+"/select?q="+query+"&rows=0&wt=json"); ClientResponse solrResource = configResource.accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); if (solrResource.getStatus() == ClientResponse.Status.OK.getStatusCode()) { @@ -119,8 +122,8 @@ public static String solrChannelPid(Client client, String ac, String fullChannel return entity; } } else { - - WebResource configResource = client.resource(fullChannelUrl+"/select?q=pid:\""+pid+"\"&rows=0&wt=json"); + String query = URLEncoder.encode( "pid:\""+pid+"\"", "UTF-8"); + WebResource configResource = client.resource(fullChannelUrl+"/select?q="+query+"\"&rows=0&wt=json"); ClientResponse solrResource = configResource.accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); if (solrResource.getStatus() == ClientResponse.Status.OK.getStatusCode()) { From 41161c445fd75c89399962f1dce1663e185f1c89 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 22 Sep 2024 00:43:25 +0200 Subject: [PATCH 219/235] Debug messages --- .../src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java index 07d91cca79..3873876e22 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java @@ -114,6 +114,7 @@ public static void checkSolrChannelEndpoint(Client client, String ac,String full public static String solrChannelPid(Client client, String ac, String fullChannelUrl, String apiVersion, String pid) throws UnsupportedEncodingException { if (apiVersion.toLowerCase().equals("v5")) { String query = URLEncoder.encode( "PID:\""+pid+"\"", "UTF-8"); + LOGGER.info("Query :"+query); WebResource configResource = client.resource(fullChannelUrl+"/select?q="+query+"&rows=0&wt=json"); ClientResponse solrResource = configResource.accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); @@ -123,6 +124,7 @@ public static String solrChannelPid(Client client, String ac, String fullChannel } } else { String query = URLEncoder.encode( "pid:\""+pid+"\"", "UTF-8"); + LOGGER.info("Query :"+query); WebResource configResource = client.resource(fullChannelUrl+"/select?q="+query+"\"&rows=0&wt=json"); ClientResponse solrResource = configResource.accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); From 4066b4c37664ee0b3311784f62e6458f8b3da76b Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 22 Sep 2024 06:49:47 +0200 Subject: [PATCH 220/235] query URLEncoder --- .../rest/apiNew/admin/v10/ItemsResource.java | 33 +++++++++++-------- .../cz/incad/kramerius/cdk/ChannelUtils.java | 10 +++--- 2 files changed, 26 insertions(+), 17 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java index 3ac12bb9bb..0fe3083052 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java @@ -18,6 +18,7 @@ import org.apache.commons.io.IOUtils; import org.dom4j.Document; import org.json.JSONArray; +import org.json.JSONException; import org.json.JSONObject; import com.google.inject.Inject; @@ -29,6 +30,7 @@ import javax.ws.rs.core.Response; import javax.ws.rs.core.StreamingOutput; import java.io.InputStream; +import java.io.UnsupportedEncodingException; import java.util.Arrays; import java.util.Iterator; import java.util.List; @@ -188,22 +190,27 @@ public Response checkItemExists(@PathParam("pid") String pid) { @Path("{pid}/solr/instintrospect") @Produces(MediaType.APPLICATION_JSON) public Response introspectPidInInstances(@PathParam("pid") String pid) { - JSONObject obj = new JSONObject(); - List instances = libraries.enabledInstances(); - for(OneInstance inst:instances) { - String library = inst.getName(); - boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + library + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + library + ".licenses") : false; - if(channelAccess) { - String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".forwardurl"); - String solrChannelUrl = ChannelUtils.solrChannelUrl(inst.getInstanceType().name(), channel); - InstanceType instType = inst.getInstanceType(); - String solrPid = ChannelUtils.solrChannelPid(this.client, channel, solrChannelUrl, instType.name(), pid); - if (solrPid != null) { - obj.put(library, new JSONObject(solrPid)); + try { + JSONObject obj = new JSONObject(); + List instances = libraries.enabledInstances(); + for(OneInstance inst:instances) { + String library = inst.getName(); + boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + library + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + library + ".licenses") : false; + if(channelAccess) { + String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".forwardurl"); + String solrChannelUrl = ChannelUtils.solrChannelUrl(inst.getInstanceType().name(), channel); + InstanceType instType = inst.getInstanceType(); + String solrPid = ChannelUtils.solrChannelPid(this.client, channel, solrChannelUrl, instType.name(), pid); + if (solrPid != null) { + obj.put(library, new JSONObject(solrPid)); + } } } + return Response.ok(obj.toString()).build(); + } catch (UnsupportedEncodingException | JSONException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new InternalErrorException(e.getMessage()); } - return Response.ok(obj.toString()).build(); } diff --git a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java index 3873876e22..cd7f9d63f6 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java @@ -114,8 +114,9 @@ public static void checkSolrChannelEndpoint(Client client, String ac,String full public static String solrChannelPid(Client client, String ac, String fullChannelUrl, String apiVersion, String pid) throws UnsupportedEncodingException { if (apiVersion.toLowerCase().equals("v5")) { String query = URLEncoder.encode( "PID:\""+pid+"\"", "UTF-8"); - LOGGER.info("Query :"+query); - WebResource configResource = client.resource(fullChannelUrl+"/select?q="+query+"&rows=0&wt=json"); + String url = fullChannelUrl+"/select?q="+query+"&rows=0&wt=json"; + LOGGER.info("SOLR URL "+url); + WebResource configResource = client.resource(url); ClientResponse solrResource = configResource.accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); if (solrResource.getStatus() == ClientResponse.Status.OK.getStatusCode()) { @@ -124,8 +125,9 @@ public static String solrChannelPid(Client client, String ac, String fullChannel } } else { String query = URLEncoder.encode( "pid:\""+pid+"\"", "UTF-8"); - LOGGER.info("Query :"+query); - WebResource configResource = client.resource(fullChannelUrl+"/select?q="+query+"\"&rows=0&wt=json"); + String url = fullChannelUrl+"/select?q="+query+"\"&rows=0&wt=json"; + LOGGER.info("SOLR URL "+url); + WebResource configResource = client.resource(url); ClientResponse solrResource = configResource.accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); if (solrResource.getStatus() == ClientResponse.Status.OK.getStatusCode()) { From 7a25f242842a2669769a14409351604e82e92c25 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 22 Sep 2024 07:23:15 +0200 Subject: [PATCH 221/235] basic field list in introspection --- .../java/cz/incad/kramerius/cdk/ChannelUtils.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java index cd7f9d63f6..ebd469b3d5 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java @@ -2,8 +2,10 @@ import java.io.UnsupportedEncodingException; import java.net.URLEncoder; +import java.util.List; import java.util.Map; import java.util.logging.Logger; +import java.util.stream.Collectors; import javax.ws.rs.core.MediaType; @@ -111,11 +113,11 @@ public static void checkSolrChannelEndpoint(Client client, String ac,String full } else throw new IllegalStateException(String.format("Channel for %s(%s) doesnt work ", ac, fullChannelUrl)); } - public static String solrChannelPid(Client client, String ac, String fullChannelUrl, String apiVersion, String pid) throws UnsupportedEncodingException { + public static String solrChannelPidExistence(Client client, String ac, String fullChannelUrl, String apiVersion, String pid, List fieldList) throws UnsupportedEncodingException { + // PID, fedora.model, pid_path, root_pid, if (apiVersion.toLowerCase().equals("v5")) { String query = URLEncoder.encode( "PID:\""+pid+"\"", "UTF-8"); - String url = fullChannelUrl+"/select?q="+query+"&rows=0&wt=json"; - LOGGER.info("SOLR URL "+url); + String url = fullChannelUrl+"/select?q="+query+"&rows=0&wt=json&fl=PID,fedora.model,pid_path,root_pid"; WebResource configResource = client.resource(url); ClientResponse solrResource = configResource.accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); @@ -124,9 +126,9 @@ public static String solrChannelPid(Client client, String ac, String fullChannel return entity; } } else { + // pid, model, pid_paths, root.pid String query = URLEncoder.encode( "pid:\""+pid+"\"", "UTF-8"); - String url = fullChannelUrl+"/select?q="+query+"\"&rows=0&wt=json"; - LOGGER.info("SOLR URL "+url); + String url = fullChannelUrl+"/select?q="+query+"&rows=0&wt=json&fl=pid,model,pid_paths,root.pid"; WebResource configResource = client.resource(url); ClientResponse solrResource = configResource.accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); From 4d5aed1df596fe8de2a27e1db2b85c8c53ae4234 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 22 Sep 2024 07:30:28 +0200 Subject: [PATCH 222/235] Remove fieldList paramter, replaced with fixed parameters --- .../src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java index ebd469b3d5..b721473252 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java @@ -113,7 +113,7 @@ public static void checkSolrChannelEndpoint(Client client, String ac,String full } else throw new IllegalStateException(String.format("Channel for %s(%s) doesnt work ", ac, fullChannelUrl)); } - public static String solrChannelPidExistence(Client client, String ac, String fullChannelUrl, String apiVersion, String pid, List fieldList) throws UnsupportedEncodingException { + public static String solrChannelPidExistence(Client client, String ac, String fullChannelUrl, String apiVersion, String pid) throws UnsupportedEncodingException { // PID, fedora.model, pid_path, root_pid, if (apiVersion.toLowerCase().equals("v5")) { String query = URLEncoder.encode( "PID:\""+pid+"\"", "UTF-8"); From 3a9521ad5b3f792ca95a3d72324a1b2013da2f14 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 22 Sep 2024 07:38:47 +0200 Subject: [PATCH 223/235] ItemsResource --- .../cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java index 0fe3083052..d4fa5b182a 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java @@ -200,7 +200,7 @@ public Response introspectPidInInstances(@PathParam("pid") String pid) { String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".forwardurl"); String solrChannelUrl = ChannelUtils.solrChannelUrl(inst.getInstanceType().name(), channel); InstanceType instType = inst.getInstanceType(); - String solrPid = ChannelUtils.solrChannelPid(this.client, channel, solrChannelUrl, instType.name(), pid); + String solrPid = ChannelUtils.solrChannelPidExistence(this.client, channel, solrChannelUrl, instType.name(), pid); if (solrPid != null) { obj.put(library, new JSONObject(solrPid)); } From b8a0c1c329d1b568e58d377e08ada211f813b4e6 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 22 Sep 2024 07:55:02 +0200 Subject: [PATCH 224/235] Rows fixed --- .../src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java index b721473252..cda7a31476 100644 --- a/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java +++ b/shared/common/src/main/java/cz/incad/kramerius/cdk/ChannelUtils.java @@ -117,7 +117,7 @@ public static String solrChannelPidExistence(Client client, String ac, String fu // PID, fedora.model, pid_path, root_pid, if (apiVersion.toLowerCase().equals("v5")) { String query = URLEncoder.encode( "PID:\""+pid+"\"", "UTF-8"); - String url = fullChannelUrl+"/select?q="+query+"&rows=0&wt=json&fl=PID,fedora.model,pid_path,root_pid"; + String url = fullChannelUrl+"/select?q="+query+"&rows=1&wt=json&fl=PID,fedora.model,pid_path,root_pid"; WebResource configResource = client.resource(url); ClientResponse solrResource = configResource.accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); @@ -128,7 +128,7 @@ public static String solrChannelPidExistence(Client client, String ac, String fu } else { // pid, model, pid_paths, root.pid String query = URLEncoder.encode( "pid:\""+pid+"\"", "UTF-8"); - String url = fullChannelUrl+"/select?q="+query+"&rows=0&wt=json&fl=pid,model,pid_paths,root.pid"; + String url = fullChannelUrl+"/select?q="+query+"&rows=1&wt=json&fl=pid,model,pid_paths,root.pid"; WebResource configResource = client.resource(url); ClientResponse solrResource = configResource.accept(MediaType.APPLICATION_JSON) .get(ClientResponse.class); From 4a00fa34ad6b819e34e714dda3d88f18c33cd3a6 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Thu, 17 Oct 2024 11:08:02 +0200 Subject: [PATCH 225/235] Attributes for k5 - store if there is no mapper --- .../v10/proxy/ConnectedInfoResource.java | 14 +- .../v60/redirection/ProxyHandlerSupport.java | 138 ++++++++++++------ .../keycloack/KeycloackUserSupport.java | 21 ++- 3 files changed, 120 insertions(+), 53 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java index c850f2e6ba..aab405d836 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java @@ -204,6 +204,7 @@ public Response config(@PathParam("library") String library) { return Response.ok(config.toString()).build(); } + @GET @Path("{library}/config/channel/health") @Produces(MediaType.APPLICATION_JSON) @@ -211,24 +212,25 @@ public Response getChannelHealth(@PathParam("library") String library) { JSONObject healthObject = new JSONObject(); JSONObject channelObject = new JSONObject(); JSONObject usersObject = new JSONObject(); - + healthObject.put("channel", channelObject); healthObject.put("users", usersObject); channelHealth(library, channelObject,usersObject); - - return Response.ok(healthObject.toString()).build(); } + + + + + + private void channelHealth(String library, JSONObject channelObject, JSONObject usersObject) { boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + library + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + library + ".licenses") : false; String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".forwardurl"); - if (channelAccess) { OneInstance inst = this.libraries.find(library); - //healthObject.put("status", inst.isConnected()); - if (inst.isConnected() && StringUtils.isAnyString(channel)) { channelObject.put("enabled", true); // solr diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index bd9e0c109c..d033cb5c0b 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -7,11 +7,13 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; import java.util.ArrayList; import java.util.Arrays; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.UUID; @@ -29,22 +31,28 @@ import javax.ws.rs.core.StreamingOutput; import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.tuple.Pair; import org.json.JSONArray; +import org.json.JSONObject; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; +import com.google.common.base.Functions; +import com.google.common.collect.Lists; import com.sun.jersey.api.client.Client; import com.sun.jersey.api.client.ClientResponse; import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.SolrAccess; +import cz.incad.kramerius.cdk.ChannelUtils; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.AlreadyRegistedPidsException; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem.TypeOfReharvset; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance.InstanceType; import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.StringUtils; import cz.incad.kramerius.utils.XMLUtils; @@ -167,10 +175,15 @@ public void write(OutputStream output) throws IOException, WebApplicationExcepti public void deleteTriggeToReharvest(String pid) { if (reharvestManager != null && pid != null) { - // LOGGER.info(String.format("Registering pid %s",pid)); try { + + String cdkRootPid = null; + String cdkOwnPidPath = null; + String cdkOwnParentPid = null; +// String intropsectingPid = pid; + Document solrDataByPid = this.solrAccess.getSolrDataByPid(pid); - Element rootPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), + Element rootPidElm = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { @Override public boolean acceptElement(Element element) { @@ -180,8 +193,8 @@ public boolean acceptElement(Element element) { } return false; } - }); - + }); + cdkRootPid = rootPidElm != null ? rootPidElm.getTextContent() : null; Element ownPidPath = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { @Override @@ -192,9 +205,10 @@ public boolean acceptElement(Element element) { } return false; } - }); - - Element ownParentPid = XMLUtils.findElement(solrDataByPid.getDocumentElement(), + }); + cdkOwnPidPath = ownPidPath != null ? ownPidPath.getTextContent() : null; + + Element ownParentPidElm = XMLUtils.findElement(solrDataByPid.getDocumentElement(), new XMLUtils.ElementsFilter() { @Override public boolean acceptElement(Element element) { @@ -204,55 +218,61 @@ public boolean acceptElement(Element element) { } return false; } - }); - - Element cdkCollection = XMLUtils.findElement(solrDataByPid.getDocumentElement(), - new XMLUtils.ElementsFilter() { - @Override - public boolean acceptElement(Element element) { - if (element.getNodeName().equals("arr")) { - String fieldName = element.getAttribute("name"); - return fieldName.equals("cdk.collection"); - } - return false; - } - }); + }); + cdkOwnParentPid = ownParentPidElm != null ? ownParentPidElm.getTextContent() : null; - if (rootPid != null && ownPidPath != null && ownParentPid != null) { + if (rootPidElm != null && ownPidPath != null && ownParentPidElm != null) { String pidPath = ownPidPath.getTextContent().trim(); - String ownParentPidText = ownParentPid.getTextContent().trim(); + String ownParentPidText = ownParentPidElm.getTextContent().trim(); int index = pidPath.indexOf(ownParentPidText); if (index >= 0) { pidPath = pidPath.substring(0, index + ownParentPidText.length()).trim(); } + try { + Pair, List> pair = introspectPid(ownParentPidText); ReharvestItem alreadyRegistredItem = this.reharvestManager - .getOpenItemByPid(ownParentPid.getTextContent().trim()); + .getOpenItemByPid(ownParentPidText); if (alreadyRegistredItem == null) { - ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), - "Delete trigger|404 ", "open", ownParentPid.getTextContent().trim(), pidPath); - reharvestItem.setTypeOfReharvest(TypeOfReharvset.children); - reharvestItem.setState("waiting_for_approve"); - if (cdkCollection != null) { - // all libraries - List collections = new ArrayList<>(); - List enabledInstances = this.instances.enabledInstances(); - for (OneInstance inst : enabledInstances) { - String acronym = inst.getName(); - boolean channelAccess = KConfiguration.getInstance().getConfiguration() - .containsKey("cdk.collections.sources." + acronym + ".licenses") - ? KConfiguration.getInstance().getConfiguration().getBoolean( - "cdk.collections.sources." + acronym + ".licenses") - : false; - if (channelAccess) { - collections.add(acronym); + + Document onwParentPidDocument = this.solrAccess.getSolrDataByPid(ownParentPidElm.getTextContent().trim()); + Element cdkModelElement = onwParentPidDocument!= null ? XMLUtils.findElement(onwParentPidDocument.getDocumentElement(), + new XMLUtils.ElementsFilter() { + @Override + public boolean acceptElement(Element element) { + if (element.getNodeName().equals("str")) { + String fieldName = element.getAttribute("name"); + return fieldName.equals("model"); + } + return false; } + }) : null; + + ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger|404 ", "open", ownParentPidElm.getTextContent().trim(), pidPath); + List topLevelModels = Lists.transform(KConfiguration.getInstance().getConfiguration().getList("fedora.topLevelModels"), Functions.toStringFunction()); + LinkedHashSet uniqueModels = new LinkedHashSet<>(pair.getLeft()); + if (uniqueModels.size() == 1) { + String model = uniqueModels.iterator().next(); + if (cdkModelElement != null) { + String cdkModel = cdkModelElement.getTextContent().trim(); + if (!cdkModel.equals(model)) { + //TODO: cdk conflict + // delete - followed by reharvest + + } + } + if (topLevelModels.contains(model)) { + reharvestItem.setTypeOfReharvest(TypeOfReharvset.root); + } else { + reharvestItem.setTypeOfReharvest(TypeOfReharvset.children); } - reharvestItem.setLibraries(collections); + } else if (uniqueModels.size() > 1){ + // live conflict - reharvest dle nkp } + reharvestItem.setLibraries(pair.getRight()); + reharvestItem.setState("waiting_for_approve"); this.reharvestManager.register(reharvestItem); } - } catch (DOMException e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); } catch (AlreadyRegistedPidsException e) { @@ -269,6 +289,40 @@ public boolean acceptElement(Element element) { } } + private Pair, List> introspectPid(String pid) throws UnsupportedEncodingException { + List models = new ArrayList<>(); + List liveInstances = new ArrayList<>(); + List instances = this.instances.enabledInstances(); + for(OneInstance inst:instances) { + String library = inst.getName(); + boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + library + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + library + ".licenses") : false; + if(channelAccess) { + String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".forwardurl"); + String solrChannelUrl = ChannelUtils.solrChannelUrl(inst.getInstanceType().name(), channel); + InstanceType instType = inst.getInstanceType(); + String solrPid = ChannelUtils.solrChannelPidExistence(this.client, channel, solrChannelUrl, instType.name(), pid); + + JSONObject obj = new JSONObject(solrPid); + JSONObject responseObject = obj.getJSONObject("response"); + JSONArray docs = responseObject.getJSONArray("docs"); + if (docs.length() > 0 ) { + JSONObject doc = docs.getJSONObject(0); + switch(inst.getInstanceType()) { + case V5: + models.add(doc.optString("fedora.model")); + liveInstances.add(inst.getName()); + break; + case V7: + models.add(doc.optString("model")); + liveInstances.add(inst.getName()); + break; + } + } + } + } + return Pair.of(models, liveInstances); + } + protected void mockSession() { if (!user.getSessionAttributes().containsKey("shib-session-id")) { this.user.addSessionAttribute("shib-session-id", "_dd68cbd66641c9b647b05509ac0241fa"); diff --git a/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java b/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java index 3a2238a69c..8a3f5ab1d7 100644 --- a/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java +++ b/security/authfilters/src/main/java/cz/incad/kramerius/auth/thirdparty/keycloack/KeycloackUserSupport.java @@ -157,12 +157,23 @@ protected Keycloack3rdUser createUserWrapper(HttpServletRequest req, String user LOGGER.fine("Token id: "+token.getId() +", and returned claims:"+ token.getOtherClaims()); Map otherClaims = token.getOtherClaims(); otherClaims.keySet().forEach(key-> { - Object object = otherClaims.get(key); - if (object != null) { - LOGGER.log(Level.FINE,"Key value "+key+" = "+object.toString()); - keycloack3rdUser.setProperty(key, object.toString()); - } + Object object = otherClaims.get(key); + if (object != null) { + LOGGER.log(Level.FINE,"Key value "+key+" = "+object.toString()); + keycloack3rdUser.setProperty(key, object.toString()); + } }); + + /** K5 instance */ + Set allKeys = keycloack3rdUser.getPropertyKeys(); + if (allKeys.contains("eduPersonScopedAffiliation") && !allKeys.contains("affiliation")) { + keycloack3rdUser.setProperty("affiliation", keycloack3rdUser.getProperty("eduPersonScopedAffiliation")); + } + + if (allKeys.contains("eduPersonEntitlement") && !allKeys.contains("entitlement")) { + keycloack3rdUser.setProperty("entitlement", keycloack3rdUser.getProperty("eduPersonEntitlement")); + } + return keycloack3rdUser; } From 10652acf17af83c8e3cdfceab234b1d801a142db Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Sun, 20 Oct 2024 19:50:49 +0200 Subject: [PATCH 226/235] IntrospectUtils --- .../v10/proxy/ConnectedInfoResource.java | 24 +++++++- .../rest/apiNew/client/v60/ItemsResource.java | 2 +- .../v60/redirection/ProxyHandlerSupport.java | 44 +-------------- .../redirection/utils/IntrospectUtils.java | 56 +++++++++++++++++++ 4 files changed, 82 insertions(+), 44 deletions(-) create mode 100644 rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/utils/IntrospectUtils.java diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java index aab405d836..7e358b26e9 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/proxy/ConnectedInfoResource.java @@ -1,6 +1,7 @@ package cz.incad.kramerius.rest.apiNew.admin.v10.proxy; import java.io.IOException; +import java.io.UnsupportedEncodingException; import java.util.Arrays; import java.util.Collections; import java.util.Date; @@ -36,6 +37,7 @@ import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance.TypeOfChangedStatus; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.user.V5ForwardUserHandler; import cz.incad.kramerius.rest.apiNew.client.v60.redirection.user.V7ForwardUserHandler; +import cz.incad.kramerius.rest.apiNew.client.v60.redirection.utils.IntrospectUtils; import cz.incad.kramerius.rest.apiNew.client.v60.libs.PhysicalLocationMap; import cz.incad.kramerius.rest.apiNew.exceptions.ForbiddenException; import cz.incad.kramerius.rest.apiNew.exceptions.InternalErrorException; @@ -221,10 +223,30 @@ public Response getChannelHealth(@PathParam("library") String library) { } + @GET + @Path("introspect/{pid}") + @Produces(MediaType.APPLICATION_JSON) + public Response introspectPid(@PathParam("pid") String pid) { + try { + Pair, List> intropsected = IntrospectUtils.introspectPid(this.client, this.libraries, pid); + JSONObject retval = new JSONObject(); + JSONArray modelsArr = new JSONArray(); + JSONArray libsArr = new JSONArray(); + intropsected.getLeft().forEach(modelsArr::put); + intropsected.getRight().forEach(libsArr::put); + retval.put("models", modelsArr); + retval.put("libraries", libsArr); + return Response.ok(retval.toString()).build(); + } catch (UnsupportedEncodingException e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build(); + } + } - + + private void channelHealth(String library, JSONObject channelObject, JSONObject usersObject) { boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + library + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + library + ".licenses") : false; diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java index 98b9d522e0..6699ed7a55 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java @@ -508,7 +508,7 @@ public Response getMetadataDublinCore(@PathParam("pid") String pid) { checkSupportedObjectPid(pid); ProxyItemHandler redirectHandler = findRedirectHandler(pid, null); if (redirectHandler != null) { - return redirectHandler.mods(get); + return redirectHandler.dc(get); } else { return Response.ok().build(); } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index d033cb5c0b..dfa8b21a5b 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -7,11 +7,9 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.io.UnsupportedEncodingException; import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; -import java.util.ArrayList; import java.util.Arrays; import java.util.LinkedHashSet; import java.util.List; @@ -32,8 +30,6 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.tuple.Pair; -import org.json.JSONArray; -import org.json.JSONObject; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -45,14 +41,12 @@ import com.sun.jersey.api.client.WebResource; import cz.incad.kramerius.SolrAccess; -import cz.incad.kramerius.cdk.ChannelUtils; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.AlreadyRegistedPidsException; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestItem.TypeOfReharvset; import cz.incad.kramerius.rest.apiNew.admin.v10.reharvest.ReharvestManager; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; -import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; -import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance.InstanceType; +import cz.incad.kramerius.rest.apiNew.client.v60.redirection.utils.IntrospectUtils; import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.StringUtils; import cz.incad.kramerius.utils.XMLUtils; @@ -230,7 +224,7 @@ public boolean acceptElement(Element element) { } try { - Pair, List> pair = introspectPid(ownParentPidText); + Pair, List> pair = IntrospectUtils.introspectPid(this.client, this.instances, ownParentPidText); ReharvestItem alreadyRegistredItem = this.reharvestManager .getOpenItemByPid(ownParentPidText); if (alreadyRegistredItem == null) { @@ -289,40 +283,6 @@ public boolean acceptElement(Element element) { } } - private Pair, List> introspectPid(String pid) throws UnsupportedEncodingException { - List models = new ArrayList<>(); - List liveInstances = new ArrayList<>(); - List instances = this.instances.enabledInstances(); - for(OneInstance inst:instances) { - String library = inst.getName(); - boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + library + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + library + ".licenses") : false; - if(channelAccess) { - String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".forwardurl"); - String solrChannelUrl = ChannelUtils.solrChannelUrl(inst.getInstanceType().name(), channel); - InstanceType instType = inst.getInstanceType(); - String solrPid = ChannelUtils.solrChannelPidExistence(this.client, channel, solrChannelUrl, instType.name(), pid); - - JSONObject obj = new JSONObject(solrPid); - JSONObject responseObject = obj.getJSONObject("response"); - JSONArray docs = responseObject.getJSONArray("docs"); - if (docs.length() > 0 ) { - JSONObject doc = docs.getJSONObject(0); - switch(inst.getInstanceType()) { - case V5: - models.add(doc.optString("fedora.model")); - liveInstances.add(inst.getName()); - break; - case V7: - models.add(doc.optString("model")); - liveInstances.add(inst.getName()); - break; - } - } - } - } - return Pair.of(models, liveInstances); - } - protected void mockSession() { if (!user.getSessionAttributes().containsKey("shib-session-id")) { this.user.addSessionAttribute("shib-session-id", "_dd68cbd66641c9b647b05509ac0241fa"); diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/utils/IntrospectUtils.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/utils/IntrospectUtils.java new file mode 100644 index 0000000000..acbf9df4cd --- /dev/null +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/utils/IntrospectUtils.java @@ -0,0 +1,56 @@ +package cz.incad.kramerius.rest.apiNew.client.v60.redirection.utils; + +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang3.tuple.Pair; +import org.json.JSONArray; +import org.json.JSONObject; + +import com.sun.jersey.api.client.Client; + +import cz.incad.kramerius.cdk.ChannelUtils; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; +import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance.InstanceType; +import cz.incad.kramerius.utils.conf.KConfiguration; + +public class IntrospectUtils { + + private IntrospectUtils() {} + + public static Pair, List> introspectPid(Client client, Instances instancesObject, String pid) throws UnsupportedEncodingException { + List models = new ArrayList<>(); + List liveInstances = new ArrayList<>(); + List instances = instancesObject.enabledInstances(); + for(OneInstance inst:instances) { + String library = inst.getName(); + boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + library + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + library + ".licenses") : false; + if(channelAccess) { + String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".forwardurl"); + String solrChannelUrl = ChannelUtils.solrChannelUrl(inst.getInstanceType().name(), channel); + InstanceType instType = inst.getInstanceType(); + String solrPid = ChannelUtils.solrChannelPidExistence(client, channel, solrChannelUrl, instType.name(), pid); + + JSONObject obj = new JSONObject(solrPid); + JSONObject responseObject = obj.getJSONObject("response"); + JSONArray docs = responseObject.getJSONArray("docs"); + if (docs.length() > 0 ) { + JSONObject doc = docs.getJSONObject(0); + switch(inst.getInstanceType()) { + case V5: + models.add(doc.optString("fedora.model")); + liveInstances.add(inst.getName()); + break; + case V7: + models.add(doc.optString("model")); + liveInstances.add(inst.getName()); + break; + } + } + } + } + return Pair.of(models, liveInstances); + } +} From 001bc0dccadc91ac280b5bb727b6b0e5e4fd16e7 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 22 Oct 2024 09:37:18 +0200 Subject: [PATCH 227/235] solr introspect; K5 support --- .../rest/apiNew/admin/v10/ItemsResource.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java index d4fa5b182a..d46e93973a 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java @@ -202,6 +202,36 @@ public Response introspectPidInInstances(@PathParam("pid") String pid) { InstanceType instType = inst.getInstanceType(); String solrPid = ChannelUtils.solrChannelPidExistence(this.client, channel, solrChannelUrl, instType.name(), pid); if (solrPid != null) { + JSONObject responseObj = null; + switch(instType) { + case V5: + // make solr fields accessible + // PID copy to pid + // fedora.model copy to model + // root_pid copy to root_pid + // pid_path copy to pid_paths + JSONObject k5resp = new JSONObject(solrPid); + JSONObject optJSONObject = k5resp.optJSONObject("response"); + if (optJSONObject != null) { + JSONArray docs = optJSONObject.getJSONArray("docs"); + for (int i = 0; i < docs.length(); i++) { + JSONObject doc = docs.getJSONObject(i); + if (doc.has("PID")) { + doc.put("pid", doc.getString("pid")); + } + if (doc.has("fedora.model")) { + doc.put("model", doc.getString("fedora.model")); + } + + if (doc.has("pid_path")) { + doc.put("pid_paths", doc.getJSONArray("pid_path")); + } +} + } + + default: + responseObj = new JSONObject(solrPid); + } obj.put(library, new JSONObject(solrPid)); } } From a88403d1f2f123be0fa6421b7315b01a09d9d0f3 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 22 Oct 2024 09:59:16 +0200 Subject: [PATCH 228/235] pid -> PID --- .../cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java index d46e93973a..cc216fdb7b 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java @@ -217,7 +217,7 @@ public Response introspectPidInInstances(@PathParam("pid") String pid) { for (int i = 0; i < docs.length(); i++) { JSONObject doc = docs.getJSONObject(i); if (doc.has("PID")) { - doc.put("pid", doc.getString("pid")); + doc.put("pid", doc.getString("PID")); } if (doc.has("fedora.model")) { doc.put("model", doc.getString("fedora.model")); From 4bdf42489b7940bb591ff76e0559745620df20c1 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 22 Oct 2024 10:15:19 +0200 Subject: [PATCH 229/235] introspect solr endpoint --- .../incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java index cc216fdb7b..b8e61ad23b 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java @@ -228,11 +228,14 @@ public Response introspectPidInInstances(@PathParam("pid") String pid) { } } } + responseObj = k5resp; + break; default: responseObj = new JSONObject(solrPid); + break; } - obj.put(library, new JSONObject(solrPid)); + obj.put(library, responseObj); } } } From ccc3327983cf5b66f319612aa9d13b43945522dd Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 22 Oct 2024 10:29:44 +0200 Subject: [PATCH 230/235] root_pid => root.pid --- .../kramerius/rest/apiNew/admin/v10/ItemsResource.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java index b8e61ad23b..3c04d094d3 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java @@ -220,12 +220,16 @@ public Response introspectPidInInstances(@PathParam("pid") String pid) { doc.put("pid", doc.getString("PID")); } if (doc.has("fedora.model")) { - doc.put("model", doc.getString("fedora.model")); + doc.put("model", doc.root_pid("fedora.model")); } if (doc.has("pid_path")) { doc.put("pid_paths", doc.getJSONArray("pid_path")); } + + if (doc.has("root_pid")) { + doc.put("root.pid", doc.root_pid("root_pid")); + } } } responseObj = k5resp; From 55d8308d37f02ccdb8b13fbef40b8ce3c7362211 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Tue, 22 Oct 2024 10:39:39 +0200 Subject: [PATCH 231/235] root_pid => root.pid --- .../incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java index 3c04d094d3..ab7933e06b 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java @@ -220,7 +220,7 @@ public Response introspectPidInInstances(@PathParam("pid") String pid) { doc.put("pid", doc.getString("PID")); } if (doc.has("fedora.model")) { - doc.put("model", doc.root_pid("fedora.model")); + doc.put("model", doc.getString("fedora.model")); } if (doc.has("pid_path")) { @@ -228,7 +228,7 @@ public Response introspectPidInInstances(@PathParam("pid") String pid) { } if (doc.has("root_pid")) { - doc.put("root.pid", doc.root_pid("root_pid")); + doc.put("root.pid", doc.getString("root_pid")); } } } From 89a6eae8e60626cddd5c20acd5ae870e6a3b3833 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 23 Oct 2024 15:17:56 +0200 Subject: [PATCH 232/235] Delete trigger; --- .../rest/apiNew/admin/v10/ItemsResource.java | 61 +--------------- .../rest/apiNew/client/v60/ItemsResource.java | 46 ++++++------- .../v60/redirection/ProxyHandlerSupport.java | 69 ++++++++++++++++--- .../redirection/utils/IntrospectUtils.java | 57 +++++++++++++++ 4 files changed, 144 insertions(+), 89 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java index ab7933e06b..0b8689de7a 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/admin/v10/ItemsResource.java @@ -1,19 +1,16 @@ package cz.incad.kramerius.rest.apiNew.admin.v10; -import cz.incad.kramerius.cdk.ChannelUtils; import cz.incad.kramerius.repository.RepositoryApi; import cz.incad.kramerius.repository.utils.Utils; import cz.incad.kramerius.rest.api.k5.client.utils.UsersUtils; import cz.incad.kramerius.rest.apiNew.client.v60.libs.Instances; -import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance; -import cz.incad.kramerius.rest.apiNew.client.v60.libs.OneInstance.InstanceType; +import cz.incad.kramerius.rest.apiNew.client.v60.redirection.utils.IntrospectUtils; import cz.incad.kramerius.rest.apiNew.exceptions.BadRequestException; import cz.incad.kramerius.rest.apiNew.exceptions.ForbiddenException; import cz.incad.kramerius.rest.apiNew.exceptions.InternalErrorException; import cz.incad.kramerius.security.Role; import cz.incad.kramerius.security.User; import cz.incad.kramerius.utils.StringUtils; -import cz.incad.kramerius.utils.conf.KConfiguration; import cz.incad.kramerius.utils.java.Pair; import org.apache.commons.io.IOUtils; import org.dom4j.Document; @@ -191,66 +188,14 @@ public Response checkItemExists(@PathParam("pid") String pid) { @Produces(MediaType.APPLICATION_JSON) public Response introspectPidInInstances(@PathParam("pid") String pid) { try { - JSONObject obj = new JSONObject(); - List instances = libraries.enabledInstances(); - for(OneInstance inst:instances) { - String library = inst.getName(); - boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + library + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + library + ".licenses") : false; - if(channelAccess) { - String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".forwardurl"); - String solrChannelUrl = ChannelUtils.solrChannelUrl(inst.getInstanceType().name(), channel); - InstanceType instType = inst.getInstanceType(); - String solrPid = ChannelUtils.solrChannelPidExistence(this.client, channel, solrChannelUrl, instType.name(), pid); - if (solrPid != null) { - JSONObject responseObj = null; - switch(instType) { - case V5: - // make solr fields accessible - // PID copy to pid - // fedora.model copy to model - // root_pid copy to root_pid - // pid_path copy to pid_paths - JSONObject k5resp = new JSONObject(solrPid); - JSONObject optJSONObject = k5resp.optJSONObject("response"); - if (optJSONObject != null) { - JSONArray docs = optJSONObject.getJSONArray("docs"); - for (int i = 0; i < docs.length(); i++) { - JSONObject doc = docs.getJSONObject(i); - if (doc.has("PID")) { - doc.put("pid", doc.getString("PID")); - } - if (doc.has("fedora.model")) { - doc.put("model", doc.getString("fedora.model")); - } - - if (doc.has("pid_path")) { - doc.put("pid_paths", doc.getJSONArray("pid_path")); - } - - if (doc.has("root_pid")) { - doc.put("root.pid", doc.getString("root_pid")); - } -} - } - responseObj = k5resp; - break; - - default: - responseObj = new JSONObject(solrPid); - break; - } - obj.put(library, responseObj); - } - } - } + JSONObject obj = IntrospectUtils.introspectSolr(this.client, this.libraries, pid); return Response.ok(obj.toString()).build(); } catch (UnsupportedEncodingException | JSONException e) { LOGGER.log(Level.SEVERE, e.getMessage(), e); throw new InternalErrorException(e.getMessage()); } } - - + @GET @Path("{pid}/foxml") @Produces(MediaType.APPLICATION_XML) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java index 6699ed7a55..c57c083426 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java @@ -412,29 +412,29 @@ public Response getMetadataMods(@PathParam("pid") String pid) { } // -// @GET -// @Path("{pid}/metadata/deletetrig") -// @Produces(MediaType.APPLICATION_JSON) -// public Response deleteTrigger(@PathParam("pid") String pid) { -// try { -// // redirect -// checkSupportedObjectPid(pid); -// ProxyItemHandler redirectHandler = findRedirectHandler(pid, null); -// if (redirectHandler != null) { -// JSONObject obj = new JSONObject(); -// obj.put("msg",String.format("Delete trigger for %s", pid)); -// redirectHandler.deleteTriggeToReharvest(pid); -// return Response.ok(obj.toString()).type(MediaType.APPLICATION_JSON).build(); -// } else { -// return Response.status(Response.Status.BAD_REQUEST).build(); -// } -// } catch (WebApplicationException e) { -// throw e; -// } catch (Throwable e) { -// LOGGER.log(Level.SEVERE, e.getMessage(), e); -// throw new InternalErrorException(e.getMessage()); -// } -// } + @GET + @Path("{pid}/metadata/deletetrig") + @Produces(MediaType.APPLICATION_JSON) + public Response deleteTrigger(@PathParam("pid") String pid) { + try { + // redirect + checkSupportedObjectPid(pid); + ProxyItemHandler redirectHandler = findRedirectHandler(pid, null); + if (redirectHandler != null) { + JSONObject obj = new JSONObject(); + obj.put("msg",String.format("Delete trigger for %s", pid)); + redirectHandler.deleteTriggeToReharvest(pid); + return Response.ok(obj.toString()).type(MediaType.APPLICATION_JSON).build(); + } else { + return Response.status(Response.Status.BAD_REQUEST).build(); + } + } catch (WebApplicationException e) { + throw e; + } catch (Throwable e) { + LOGGER.log(Level.SEVERE, e.getMessage(), e); + throw new InternalErrorException(e.getMessage()); + } + } @GET diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java index dfa8b21a5b..a4a0987afe 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/ProxyHandlerSupport.java @@ -10,10 +10,13 @@ import java.net.MalformedURLException; import java.net.URISyntaxException; import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.logging.Level; import java.util.logging.Logger; @@ -30,6 +33,8 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.tuple.Pair; +import org.json.JSONArray; +import org.json.JSONObject; import org.w3c.dom.DOMException; import org.w3c.dom.Document; import org.w3c.dom.Element; @@ -174,7 +179,7 @@ public void deleteTriggeToReharvest(String pid) { String cdkRootPid = null; String cdkOwnPidPath = null; String cdkOwnParentPid = null; -// String intropsectingPid = pid; + Document solrDataByPid = this.solrAccess.getSolrDataByPid(pid); Element rootPidElm = XMLUtils.findElement(solrDataByPid.getDocumentElement(), @@ -224,11 +229,45 @@ public boolean acceptElement(Element element) { } try { - Pair, List> pair = IntrospectUtils.introspectPid(this.client, this.instances, ownParentPidText); - ReharvestItem alreadyRegistredItem = this.reharvestManager - .getOpenItemByPid(ownParentPidText); + + Map map = new HashMap<>(); + + JSONObject jsonResult = IntrospectUtils.introspectSolr(this.client, this.instances, ownParentPidText); + Set keys = jsonResult.keySet(); + for (Object keyObj : keys) { + String key = keyObj.toString(); + JSONObject solrResult = jsonResult.getJSONObject(key); + JSONObject response = solrResult.getJSONObject("response"); + int numFound = response.optInt("numFound"); + if (numFound > 0) { + JSONObject doc = response.getJSONArray("docs").getJSONObject(0); + map.put(key, doc); + + /* + * + * { + "fedora.model": "periodical", + "root.pid": "uuid:1c869c00-535b-11e3-9ea2-5ef3fc9ae867", + "pid_paths": [ + "uuid:1c869c00-535b-11e3-9ea2-5ef3fc9ae867" + ], + "PID": "uuid:1c869c00-535b-11e3-9ea2-5ef3fc9ae867", + "pid": "uuid:1c869c00-535b-11e3-9ea2-5ef3fc9ae867", + "model": "periodical", + "pid_path": [ + "uuid:1c869c00-535b-11e3-9ea2-5ef3fc9ae867" + ], + "root_pid": "uuid:1c869c00-535b-11e3-9ea2-5ef3fc9ae867" + * } + * + */ + + } + } + + + ReharvestItem alreadyRegistredItem = this.reharvestManager.getOpenItemByPid(ownParentPidText); if (alreadyRegistredItem == null) { - Document onwParentPidDocument = this.solrAccess.getSolrDataByPid(ownParentPidElm.getTextContent().trim()); Element cdkModelElement = onwParentPidDocument!= null ? XMLUtils.findElement(onwParentPidDocument.getDocumentElement(), new XMLUtils.ElementsFilter() { @@ -244,7 +283,21 @@ public boolean acceptElement(Element element) { ReharvestItem reharvestItem = new ReharvestItem(UUID.randomUUID().toString(), "Delete trigger|404 ", "open", ownParentPidElm.getTextContent().trim(), pidPath); List topLevelModels = Lists.transform(KConfiguration.getInstance().getConfiguration().getList("fedora.topLevelModels"), Functions.toStringFunction()); - LinkedHashSet uniqueModels = new LinkedHashSet<>(pair.getLeft()); + + LinkedHashSet uniqueRootPids = new LinkedHashSet<>(); + map.keySet().forEach(key-> { + String rootPid = map.get(key).optString("root.pid"); + uniqueRootPids.add(rootPid); + + }); + + LinkedHashSet uniqueModels = new LinkedHashSet<>(); + map.keySet().forEach(key-> { + String model = map.get(key).optString("model"); + uniqueModels.add(model); + }); + + if (uniqueModels.size() == 1) { String model = uniqueModels.iterator().next(); if (cdkModelElement != null) { @@ -252,7 +305,6 @@ public boolean acceptElement(Element element) { if (!cdkModel.equals(model)) { //TODO: cdk conflict // delete - followed by reharvest - } } if (topLevelModels.contains(model)) { @@ -263,7 +315,8 @@ public boolean acceptElement(Element element) { } else if (uniqueModels.size() > 1){ // live conflict - reharvest dle nkp } - reharvestItem.setLibraries(pair.getRight()); + reharvestItem.setLibraries(new ArrayList<>( map.keySet()) ); + //reharvestItem.setLibraries(pair.getRight()); reharvestItem.setState("waiting_for_approve"); this.reharvestManager.register(reharvestItem); } diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/utils/IntrospectUtils.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/utils/IntrospectUtils.java index acbf9df4cd..133ec47d05 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/utils/IntrospectUtils.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/redirection/utils/IntrospectUtils.java @@ -53,4 +53,61 @@ public static Pair, List> introspectPid(Client client, Inst } return Pair.of(models, liveInstances); } + + /** introspect utils */ + public static JSONObject introspectSolr(Client client, Instances libraries, String pid) throws UnsupportedEncodingException { + JSONObject obj = new JSONObject(); + List instances = libraries.enabledInstances(); + for(OneInstance inst:instances) { + String library = inst.getName(); + boolean channelAccess = KConfiguration.getInstance().getConfiguration().containsKey("cdk.collections.sources." + library + ".licenses") ? KConfiguration.getInstance().getConfiguration().getBoolean("cdk.collections.sources." + library + ".licenses") : false; + if(channelAccess) { + String channel = KConfiguration.getInstance().getConfiguration().getString("cdk.collections.sources." + library + ".forwardurl"); + String solrChannelUrl = ChannelUtils.solrChannelUrl(inst.getInstanceType().name(), channel); + InstanceType instType = inst.getInstanceType(); + String solrPid = ChannelUtils.solrChannelPidExistence(client, channel, solrChannelUrl, instType.name(), pid); + if (solrPid != null) { + JSONObject responseObj = null; + switch(instType) { + case V5: + // make solr fields accessible + // PID copy to pid + // fedora.model copy to model + // root_pid copy to root_pid + // pid_path copy to pid_paths + JSONObject k5resp = new JSONObject(solrPid); + JSONObject optJSONObject = k5resp.optJSONObject("response"); + if (optJSONObject != null) { + JSONArray docs = optJSONObject.getJSONArray("docs"); + for (int i = 0; i < docs.length(); i++) { + JSONObject doc = docs.getJSONObject(i); + if (doc.has("PID")) { + doc.put("pid", doc.getString("PID")); + } + if (doc.has("fedora.model")) { + doc.put("model", doc.getString("fedora.model")); + } + + if (doc.has("pid_path")) { + doc.put("pid_paths", doc.getJSONArray("pid_path")); + } + + if (doc.has("root_pid")) { + doc.put("root.pid", doc.getString("root_pid")); + } + } + } + responseObj = k5resp; + break; + + default: + responseObj = new JSONObject(solrPid); + break; + } + obj.put(library, responseObj); + } + } + } + return obj; + } } From e4f908b5b5795b2ccaa09a91448661df20b94faf Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 23 Oct 2024 15:39:59 +0200 Subject: [PATCH 233/235] deletetrig - commented --- .../incad/kramerius/rest/apiNew/client/v60/ItemsResource.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java index c57c083426..f70bf263d0 100644 --- a/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java +++ b/rest/src/main/java/cz/incad/kramerius/rest/apiNew/client/v60/ItemsResource.java @@ -411,7 +411,7 @@ public Response getMetadataMods(@PathParam("pid") String pid) { } } - // + /** @GET @Path("{pid}/metadata/deletetrig") @Produces(MediaType.APPLICATION_JSON) @@ -434,7 +434,7 @@ public Response deleteTrigger(@PathParam("pid") String pid) { LOGGER.log(Level.SEVERE, e.getMessage(), e); throw new InternalErrorException(e.getMessage()); } - } + }**/ @GET From f9789ab0fc4baf8785254c24bedf172a1ba64562 Mon Sep 17 00:00:00 2001 From: pavel-stastny Date: Wed, 30 Oct 2024 20:02:55 +0100 Subject: [PATCH 234/235] issue #73 --- .../kramerius/services/transform/K7SourceToDestTransform.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java index 4f5e8f5891..59369f854c 100644 --- a/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java +++ b/processes/cdkprocesses/src/main/java/cz/incad/kramerius/services/transform/K7SourceToDestTransform.java @@ -43,9 +43,13 @@ public K7SourceToDestTransform() { plainValueFields.put("keywords", Arrays.asList("keywords.search","keywords.facet")); plainValueFields.put("geographic_names", Arrays.asList("geographic_names.search","geographic_names.facet")); plainValueFields.put("search_autor", Arrays.asList("authors","authors.search")); + + plainValueFields.put("dc.creatorr", Arrays.asList("authors","authors.search")); + plainValueFields.put("facet_autor", Arrays.asList("authors.facet")); plainValueFields.put("dc.title", Arrays.asList("title.search","titles.search")); + //plainValueFields.put("dc.title", Arrays.asList("titles.search")); plainValueFields.put("keywords", Arrays.asList("keywords.search", "keywords.facet")); From ba16ad566de996818868239ff20c8002a7016f4e Mon Sep 17 00:00:00 2001 From: vlahoda Date: Thu, 2 Jan 2025 10:42:54 +0100 Subject: [PATCH 235/235] Upgraded version of JBIG2-imageio convertor, required upgrade of Guice to support Java modules... Fixes #1104 --- build.gradle | 3 +-- shared/common/build.gradle | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/build.gradle b/build.gradle index dda4736134..6ea8e8d491 100644 --- a/build.gradle +++ b/build.gradle @@ -91,7 +91,7 @@ allprojects { compileJava.options.encoding = 'UTF-8' compileTestJava.options.encoding = 'UTF-8' - ext.guiceversion ='4.2.2' + ext.guiceversion ='6.0.0' ext.postgresversion = '42.3.9' ext.slf4jversion='1.7.35' @@ -120,7 +120,6 @@ allprojects { // guice to all subprojects api ("com.google.inject:guice:${guiceversion}") {exclude group: 'com.google.guava', module: 'guava'} - api ("com.google.inject.extensions:guice-multibindings:${guiceversion}"){exclude group: 'com.google.guava', module: 'guava'} api ("com.google.inject.extensions:guice-servlet:${guiceversion}"){exclude group: 'com.google.guava', module: 'guava'} api "com.google.inject.extensions:guice-assistedinject:${guiceversion}" api "javax.inject:javax.inject:1" diff --git a/shared/common/build.gradle b/shared/common/build.gradle index 1a26b5918d..410ed7db60 100644 --- a/shared/common/build.gradle +++ b/shared/common/build.gradle @@ -18,7 +18,7 @@ dependencies { //api 'org.apache.pdfbox:pdfbox:2.0.26' api 'org.apache.pdfbox:pdfbox:2.0.32' - api 'com.levigo.jbig2:levigo-jbig2-imageio:1.6.5' + api 'com.levigo.jbig2:levigo-jbig2-imageio:2.0' api 'com.ibm.icu:icu4j:56.1' api 'com.google.guava:guava:31.0.1-jre' api 'org.bouncycastle:bcprov-jdk15on:1.68'