Merge pull request #939 from mediathekview/dev/importFilmlists

Filmlistimport & Url availability

Author: codingPF

MServer-Config.yaml

@@ -83,15 +83,30 @@ writeFilmlistIdFileEnabled: true
 #The fimlist id file path
 filmlistIdFilePath: target/filmlists/filmlist.id
 
-# Sets if a filmlist should be imported
-filmlistImporEnabled: false
 
-# The format of the film list to import.
-# Possible are: JSON, OLD_JSON, JSON_COMPRESSED_XZ, OLD_JSON_COMPRESSED_XZ, JSON_COMPRESSED_GZIP, OLD_JSON_COMPRESSED_BZIP, JSON_COMPRESSED_GZIP, OLD_JSON_COMPRESSED_BZIP
-#filmlistImportFormat: OLD_JSON_COMPRESSED_XZ
-
-# The path/URL of the film list to import.
-#filmlistImportLocation: http://verteiler1.mediathekview.de/Filmliste-akt.xz
+# import additional filmlist sources
+importFilmlistConfigurations :
+    - active: false
+      path: "someCrawlerlist.json"
+      format: OLD_JSON
+      createDiff: false
+      checkImportListUrl: false
+    - active: false
+      path: "someMoreCrawlerlist.json"
+      format: OLD_JSON
+      createDiff: false
+      checkImportListUrl: false
+    - active: false
+      path: "https://verteiler1.mediathekview.de/filme-org.xz"
+      format: OLD_JSON_COMPRESSED_XZ
+      createDiff: true
+      checkImportListUrl: true    
+
+# film url is consider invalid if the size is below the minSize
+checkImportListUrlMinSize: 5012
+
+# abort url checking after x sec
+checkImportListUrlTimeoutInSec: 1800
 
 #### Default crawler configurations ####
 # The maximum amount of URLs to be processed per task.

src/main/java/de/mediathekview/mserver/base/config/ImportFilmlistConfiguration.java

@@ -0,0 +1,76 @@
+package de.mediathekview.mserver.base.config;
+
+import java.util.Objects;
+
+import de.mediathekview.mlib.filmlisten.FilmlistFormats;
+
+public class ImportFilmlistConfiguration {
+  private final Boolean active;
+  private final String path;
+  private final FilmlistFormats format;
+  private final Boolean createDiff;
+  private final Boolean checkImportListUrl;
+  
+
+  public ImportFilmlistConfiguration(Boolean active, String path, FilmlistFormats format, Boolean createDiff, Boolean checkImportListUrl) {
+    this.active = active;
+    this.path = path;
+    this.format = format;
+    this.createDiff = createDiff;
+    this.checkImportListUrl = checkImportListUrl; 
+  }
+  
+  public ImportFilmlistConfiguration() {
+    this.active = null;
+    this.path = null;
+    this.format = null;
+    this.createDiff = null;
+    this.checkImportListUrl = null;
+  }
+  
+  public Boolean isCheckImportListUrl() {
+    return checkImportListUrl;
+  }
+  
+  public Boolean isActive() {
+    return active;
+  }
+  public String getPath() {
+    return path;
+  }
+  public FilmlistFormats getFormat() {
+    return format;
+  }
+  public Boolean isCreateDiff() {
+    return createDiff;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj) {
+      return true;
+    }
+    if (!(obj instanceof final ImportFilmlistConfiguration that)) {
+      return false;
+    }
+    if (!super.equals(obj)) {
+      return false;
+    }
+    return Objects.equals(isActive(), that.isActive())
+        && Objects.equals(getPath(), that.getPath())
+        && Objects.equals(getFormat(), that.getFormat())
+        && Objects.equals(isCreateDiff(), that.isCreateDiff()
+        && Objects.equals(isCheckImportListUrl(), that.isCheckImportListUrl()));
+  }
+  
+  @Override
+  public int hashCode() {
+    return Objects.hash(
+        super.hashCode(),
+        isActive(),
+        getPath(),
+        getFormat(),
+        isCreateDiff(),
+        isCheckImportListUrl());
+  }
+}

src/main/java/de/mediathekview/mserver/base/config/MServerConfigDTO.java

@@ -18,26 +18,28 @@ public class MServerConfigDTO extends MServerBasicConfigDTO implements ConfigDTO
   private final String ignoreFilmlistPath;
   /** add livestreams from external list **/
   private final ImportLivestreamConfiguration importLivestreamConfiguration;
+  /** add additional filmlist from external **/
+  private final List<ImportFilmlistConfiguration> importFilmlistConfigurations;
   /** The maximum amount of cpu threads to be used. */
   private Integer maximumCpuThreads;
   /**
    * The maximum duration in minutes the server should run.<br>
    * If set to 0 the server runs without a time limit.
    */
   private Integer maximumServerDurationInMinutes;
-
+  private Long checkImportListUrlMinSize;
+  private Long checkImportListUrlTimeoutInSec;
+  
   private Map<Sender, MServerBasicConfigDTO> senderConfigurations;
   private Set<Sender> senderExcluded;
   private Set<Sender> senderIncluded;
   private Set<FilmlistFormats> filmlistSaveFormats;
   private Map<FilmlistFormats, String> filmlistSavePaths;
   private Map<FilmlistFormats, String> filmlistDiffSavePaths;
-  private FilmlistFormats filmlistImportFormat;
   private String filmlistImportLocation;
   private MServerLogSettingsDTO logSettings;
   private Map<CrawlerUrlType, URL> crawlerURLs;
   private Map<CrawlerApiParam, String> crawlerApiParams;
-  private Boolean filmlistImporEnabled;
 
   public MServerConfigDTO() {
     super();
@@ -61,6 +63,9 @@ public MServerConfigDTO() {
 
     maximumCpuThreads = 80;
     maximumServerDurationInMinutes = 0;
+    checkImportListUrlMinSize = 2048L;
+    checkImportListUrlTimeoutInSec = 3600L;
+    
     filmlistSaveFormats.add(FilmlistFormats.JSON);
     filmlistSaveFormats.add(FilmlistFormats.OLD_JSON);
     filmlistSaveFormats.add(FilmlistFormats.JSON_COMPRESSED_XZ);
@@ -79,16 +84,13 @@ public MServerConfigDTO() {
     filmlistSavePaths.put(FilmlistFormats.JSON_COMPRESSED_BZIP, "filmliste.json.bz");
     filmlistSavePaths.put(FilmlistFormats.OLD_JSON_COMPRESSED_BZIP, "filmliste_old.json.bz");
 
-    filmlistImporEnabled = true;
-    filmlistImportFormat = FilmlistFormats.OLD_JSON_COMPRESSED_XZ;
-    filmlistImportLocation = "https://verteiler1.mediathekview.de/Filmliste-akt.xz";
-
     writeFilmlistHashFileEnabled = false;
     filmlistHashFilePath = "filmlist.hash";
     writeFilmlistIdFileEnabled = true;
     filmlistIdFilePath = "filmlist.id";
     ignoreFilmlistPath = "ignoreFilmlist.txt";
     importLivestreamConfiguration = new ImportLivestreamConfiguration(false, "live-streams.json", FilmlistFormats.OLD_JSON);
+    importFilmlistConfigurations = new ArrayList<>();
 
     Arrays.stream(Sender.values())
         .forEach(sender -> senderConfigurations.put(sender, new MServerBasicConfigDTO(this)));
@@ -122,14 +124,6 @@ public void setFilmlistDiffSavePaths(final Map<FilmlistFormats, String> filmlist
     this.filmlistDiffSavePaths = filmlistDiffSavePaths;
   }
 
-  public FilmlistFormats getFilmlistImportFormat() {
-    return filmlistImportFormat;
-  }
-
-  public void setFilmlistImportFormat(final FilmlistFormats filmlistImportFormat) {
-    this.filmlistImportFormat = filmlistImportFormat;
-  }
-
   public String getFilmlistImportLocation() {
     return filmlistImportLocation;
   }
@@ -177,6 +171,22 @@ public Integer getMaximumServerDurationInMinutes() {
   public void setMaximumServerDurationInMinutes(final Integer aMaximumServerDurationInMinutes) {
     maximumServerDurationInMinutes = aMaximumServerDurationInMinutes;
   }
+  
+  public Long getCheckImportListUrlMinSize() {
+    return checkImportListUrlMinSize;
+  }
+
+  public void setCheckImportListUrlMinSize(final Long checkImportListUrlMinSize) {
+    this.checkImportListUrlMinSize = checkImportListUrlMinSize;
+  }
+
+  public Long getCheckImportListUrlTimeoutInSec() {
+    return checkImportListUrlTimeoutInSec;
+  }
+
+  public void setCheckImportListUrlTimeoutInSec(final Long checkImportListUrlTimeoutInSec) {
+    this.checkImportListUrlTimeoutInSec = checkImportListUrlTimeoutInSec;
+  }
 
   public Map<Sender, MServerBasicConfigDTO> getSenderConfigurations() {
     return senderConfigurations;
@@ -242,6 +252,9 @@ public ImportLivestreamConfiguration getImportLivestreamConfiguration() {
     return importLivestreamConfiguration;
   }
 
+  public List<ImportFilmlistConfiguration> getImportFilmlistConfigurations() {
+    return importFilmlistConfigurations;
+  }
 
   /**
    * Loads the {@link Sender} specific configuration and if it not exist creates one.
@@ -255,14 +268,6 @@ public MServerBasicConfigDTO getSenderConfig(final Sender aSender) {
     return senderConfigurations.get(aSender);
   }
 
-  public Boolean getFilmlistImporEnabled() {
-    return filmlistImporEnabled;
-  }
-
-  public void setFilmlistImporEnabled(final Boolean filmlistImporEnabled) {
-    this.filmlistImporEnabled = filmlistImporEnabled;
-  }
-
   @Override
   public boolean equals(final Object o) {
     if (this == o) {
@@ -276,25 +281,25 @@ public boolean equals(final Object o) {
     }
     return Objects.equals(getCopySettings(), that.getCopySettings())
         && Objects.equals(getMaximumCpuThreads(), that.getMaximumCpuThreads())
-        && Objects.equals(
-            getMaximumServerDurationInMinutes(), that.getMaximumServerDurationInMinutes())
+        && Objects.equals(getMaximumServerDurationInMinutes(), that.getMaximumServerDurationInMinutes())
+        && Objects.equals(getCheckImportListUrlMinSize(), that.getCheckImportListUrlMinSize())
+        && Objects.equals(getCheckImportListUrlTimeoutInSec(), that.getCheckImportListUrlTimeoutInSec())
         && Objects.equals(senderConfigurations, that.senderConfigurations)
         && Objects.equals(getSenderExcluded(), that.getSenderExcluded())
         && Objects.equals(getSenderIncluded(), that.getSenderIncluded())
         && Objects.equals(getFilmlistSaveFormats(), that.getFilmlistSaveFormats())
         && Objects.equals(getFilmlistSavePaths(), that.getFilmlistSavePaths())
         && Objects.equals(getFilmlistDiffSavePaths(), that.getFilmlistDiffSavePaths())
-        && getFilmlistImportFormat() == that.getFilmlistImportFormat()
         && Objects.equals(getFilmlistImportLocation(), that.getFilmlistImportLocation())
         && Objects.equals(getLogSettings(), that.getLogSettings())
         && Objects.equals(getCrawlerURLs(), that.getCrawlerURLs())
-        && Objects.equals(getFilmlistImporEnabled(), that.getFilmlistImporEnabled())
         && Objects.equals(getWriteFilmlistHashFileEnabled(), that.getWriteFilmlistHashFileEnabled())
         && Objects.equals(getFilmlistHashFilePath(), that.getFilmlistHashFilePath())
         && Objects.equals(getWriteFilmlistIdFileEnabled(), that.getWriteFilmlistIdFileEnabled())
         && Objects.equals(getFilmlistIdFilePath(), that.getFilmlistIdFilePath())
         && Objects.equals(getIgnoreFilmslistPath(), that.getIgnoreFilmslistPath())
-        && Objects.equals(getImportLivestreamConfiguration(), that.getImportLivestreamConfiguration());
+        && Objects.equals(getImportLivestreamConfiguration(), that.getImportLivestreamConfiguration())
+        && Objects.equals(getImportFilmlistConfigurations(), that.getImportFilmlistConfigurations());
   }
 
   @Override
@@ -304,23 +309,24 @@ public int hashCode() {
         getCopySettings(),
         getMaximumCpuThreads(),
         getMaximumServerDurationInMinutes(),
+        getCheckImportListUrlMinSize(),
+        getCheckImportListUrlTimeoutInSec(),
         senderConfigurations,
         getSenderExcluded(),
         getSenderIncluded(),
         getFilmlistSaveFormats(),
         getFilmlistSavePaths(),
         getFilmlistDiffSavePaths(),
-        getFilmlistImportFormat(),
         getFilmlistImportLocation(),
         getLogSettings(),
         getCrawlerURLs(),
-        getFilmlistImporEnabled(),
         getWriteFilmlistHashFileEnabled(),
         getFilmlistHashFilePath(),
         getWriteFilmlistIdFileEnabled(),
         getFilmlistIdFilePath(),
         getIgnoreFilmslistPath(),
-        getImportLivestreamConfiguration());
+        getImportLivestreamConfiguration(),
+        getImportFilmlistConfigurations());
   }
 
   public void initializeSenderConfigurations() {

src/main/java/de/mediathekview/mserver/base/messages/ServerMessages.java

@@ -27,6 +27,9 @@ public enum ServerMessages implements Message {
   NO_FILMLIST_IMPORT_FORMAT_IN_CONFIG("noFilmlistImportFormatInConfig", MessageTypes.FATAL_ERROR),
   NO_FILMLIST_IMPORT_LOCATION_IN_CONFIG(
       "noFilmlistImportLocationInConfig", MessageTypes.FATAL_ERROR),
+  NO_FILMLIST_IMPORT_ACTIVE_IN_CONFIG("noFilmlistImportActgiveInConfig", MessageTypes.FATAL_ERROR),
+  NO_FILMLIST_IMPORT_DIFF_IN_CONFIG("noFilmlistImportCreateDiffInConfig", MessageTypes.FATAL_ERROR),
+  NO_FILMLIST_IMPORT_CHECK_IN_CONFIG("noFilmlistImportCheckUrlInConfig", MessageTypes.FATAL_ERROR),
   FORMAT_NOT_IN_SAVE_FORMATS("formatNotInSaveFormats", MessageTypes.ERROR),
   UI_TO_MANY_ARGUMENTS("uiToManyArguments", MessageTypes.ERROR),
   UI_UNKNOWN_ARGUMENT("uiUnknownArgument", MessageTypes.FATAL_ERROR),