Skip to content

Commit

Permalink
Merge branch 'release/2.5.3'
Browse files Browse the repository at this point in the history 
Crawler Laufzeitprobleme behoben
fixed #68
  • Loading branch information
@alex1702
alex1702 committed Feb 14, 2017
2 parents 9819159 + b0e955d commit c692838
Show file tree
Hide file tree
Showing 4 changed files with 96 additions and 52 deletions.
2 changes: 1 addition & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ import org.apache.tools.ant.filters.ReplaceTokens
sourceCompatibility = 1.8
targetCompatibility = 1.8
group = 'de.mediathekview'
version = '2.5.2'
version = '2.5.3'

def jarName = 'MServer.jar'
def mainClass = 'mServer.Main'
Expand Down
134 changes: 87 additions & 47 deletions src/main/java/mServer/crawler/AddToFilmlist.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,35 +5,29 @@
*/
package mServer.crawler;

import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.HashSet;
import java.util.Iterator;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.stream.Collectors;
import java.util.concurrent.atomic.AtomicInteger;

import mSearch.Config;
import mSearch.daten.DatenFilm;
import mSearch.daten.ListeFilme;
import mSearch.tool.Duration;
import mSearch.tool.FileSize;
import mSearch.tool.Log;



/**
*
* @author emil
*/
public class AddToFilmlist {

private static final int MIN_SIZE_ADD_OLD = 5;

final int COUNTER_MAX = 20;
int counter = 0;
AtomicInteger threadCounter = new AtomicInteger(0);
AtomicInteger treffer = new AtomicInteger(0);
ListeFilme vonListe;
ListeFilme listeEinsortieren;
Collection<DatenFilm> filteredOnline = new ArrayList<>();

public AddToFilmlist(ListeFilme vonListe, ListeFilme listeEinsortieren) {
this.vonListe = vonListe;
Expand Down Expand Up @@ -61,68 +55,114 @@ public synchronized void addLiveStream() {
public synchronized int addOldList() {
// in eine vorhandene Liste soll eine andere Filmliste einsortiert werden
// es werden nur Filme die noch nicht vorhanden sind, einsortiert
counter = 0;
threadCounter = new AtomicInteger(0);
treffer = new AtomicInteger(0);
int size = listeEinsortieren.size();
HashSet<String> hash = new HashSet<>(listeEinsortieren.size() + 1, 1);

Duration.staticPing("AddOld-1");

// ==============================================
// nach "Thema-Titel" suchen
Collection<DatenFilm> filteredTopicTitle = new CopyOnWriteArrayList<>();
filteredTopicTitle.addAll(listeEinsortieren.parallelStream()
.filter(film -> (null == vonListe.istInFilmListe(film.arr[DatenFilm.FILM_SENDER], film.arr[DatenFilm.FILM_THEMA], film.arr[DatenFilm.FILM_TITEL])))
.collect(Collectors.toList()));

vonListe.stream().forEach((f) -> hash.add(f.getIndexAddOld()));
listeEinsortieren.removeIf((f) -> hash.contains(f.getIndexAddOld()));
hash.clear();

Log.sysLog("===== Liste einsortieren Hash =====");
Log.sysLog("Liste einsortieren, Anzahl: " + listeEinsortieren.size());
Log.sysLog("Liste einsortieren, entfernt: " + (listeEinsortieren.size() - filteredTopicTitle.size()));
Log.sysLog("Liste einsortieren, Anzahl: " + size);
Log.sysLog("Liste einsortieren, entfernt: " + (size - listeEinsortieren.size()));
Log.sysLog("Liste einsortieren, noch einsortieren: " + listeEinsortieren.size());
Log.sysLog("");
size = listeEinsortieren.size();

// ==============================================
// nach "URL" suchen
Collection<DatenFilm> filteredUrl = new CopyOnWriteArrayList<>();

Collection<String> filmUrls = vonListe.parallelStream()
.map(DatenFilm::getUrl)
.collect(Collectors.toList());

int size = filteredTopicTitle.size();

filteredUrl.addAll(filteredTopicTitle.parallelStream()
.filter(film -> !filmUrls.contains(DatenFilm.getUrl(film)))
.collect(Collectors.toList()));
vonListe.stream().forEach((f) -> hash.add(DatenFilm.getUrl(f)));
listeEinsortieren.removeIf((f) -> hash.contains(DatenFilm.getUrl(f)));
hash.clear();

Log.sysLog("===== Liste einsortieren URL =====");
Log.sysLog("Liste einsortieren, Anzahl: " + size);
Log.sysLog("Liste einsortieren, entfernt: " + (size - filteredUrl.size()));
Log.sysLog("Liste einsortieren, entfernt: " + (size - listeEinsortieren.size()));
Log.sysLog("Liste einsortieren, noch einsortieren: " + listeEinsortieren.size());
Log.sysLog("");

Duration.staticPing("AddOld-2");

int count = 0;
final int MAX_THREAD = 30;
final int MAX_WAIT_TIME = 450; // 450*2=900s -> 15 Minuten
//final int MAX_WAIT_TIME = 10; // 10*2=20s
stopOld = false;
size = listeEinsortieren.size();

// Rest nehmen wir wenn noch online
// Prüfung auf online erst am Ende durchführen, damit jeder Film nur einmalig geprüft wird
Collection<DatenFilm> filteredOnline = new CopyOnWriteArrayList<>();
filteredOnline.addAll(filteredUrl.parallelStream().filter(f ->
!Config.getStop() && FileSize.laengeLong(f.arr[DatenFilm.FILM_URL]) > MIN_SIZE_ADD_OLD
)
.collect(Collectors.toList()));
filteredOnline.parallelStream().forEach(f ->{
if(!Config.getStop())
{
initFilm(f);
for (int i = 0; i < MAX_THREAD; ++i) {
new Thread(new AddOld(listeEinsortieren)).start();
}

while (!Config.getStop() && threadCounter.get() > 0) {
try {
System.out.println("sek.: " + 2 * (count++) + " Liste: " + listeEinsortieren.size() + " Treffer: " + treffer.get() + " Threads: " + threadCounter);
if (count > MAX_WAIT_TIME) {
// dann haben wir mehr als 10 Minuten und: Stop
Log.sysLog("===== Liste einsortieren: ABBRUCH =====");
Log.sysLog("COUNT_MAX erreicht [s]: " + MAX_WAIT_TIME * 2);
Log.sysLog("");
stopOld = true;
}
wait(2000);
} catch (Exception ex) {
Log.errorLog(978451205, ex, "Fehler beim Import Old");
}
});
}

vonListe.addAll(filteredOnline);



Log.sysLog("===== Liste einsortieren: Noch online =====");
Log.sysLog("Liste einsortieren, Anzahl: " + filteredOnline.size());
Log.sysLog("Liste einsortieren, entfernt: " + (filteredOnline.size() - treffer.get()));
Log.sysLog("Liste einsortieren, Anzahl: " + size);
Log.sysLog("Liste einsortieren, entfernt: " + (size - treffer.get()));
Log.sysLog("");
Log.sysLog("In Liste einsortiert: " + treffer.get());
Log.sysLog("");
return treffer.get();
}

private void initFilm(DatenFilm film) {
private boolean stopOld = false;

private class AddOld implements Runnable {

private DatenFilm film;
private final ListeFilme listeOld;
private final int MIN_SIZE_ADD_OLD = 5; //REST eh nur Trailer

public AddOld(ListeFilme listeOld) {
this.listeOld = listeOld;
threadCounter.incrementAndGet();
}

@Override
public void run() {
while (!stopOld && (film = popOld(listeOld)) != null) {
long size = FileSize.laengeLong(film.arr[DatenFilm.FILM_URL]);
if (size > MIN_SIZE_ADD_OLD) {
addOld(film);
}
}
threadCounter.decrementAndGet();
}
}

private synchronized DatenFilm popOld(ListeFilme listeOld) {
if (listeOld.size() > 0) {
return listeOld.remove(0);
}
return null;
}

private synchronized boolean addOld(DatenFilm film) {
treffer.getAndIncrement();
film.init();
return filteredOnline.add(film);
}

}
8 changes: 6 additions & 2 deletions src/main/java/mServer/search/MserverSearch.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import mSearch.tool.Log;
import mServer.crawler.Crawler;
import mServer.crawler.CrawlerConfig;
import static mServer.crawler.CrawlerTool.loadLongMax;
import mServer.daten.MserverSearchTask;
import mServer.tool.MserverDaten;
import mServer.tool.MserverDatumZeit;
Expand Down Expand Up @@ -109,8 +110,11 @@ public boolean filmeSuchen(MserverSearchTask aktSearchTask) {
//und jetzt STOPPEN!!!!!!!!
crawler.stop();
}

t.join(20 * 60 * 1000); // 20 Minuten warten, das Erstellen/Komprimieren der Liste dauert
int w = 20 * 60 * 1000; // 20 Minuten warten, das Erstellen/Komprimieren der Liste dauert
if (loadLongMax()) {
w = 30 * 60 * 1000; // 30 Minuten bei langen Läufen
}
t.join(w);
if (t.isAlive()) {
MserverLog.systemMeldung("");
MserverLog.systemMeldung("");
Expand Down
4 changes: 2 additions & 2 deletions src/main/resources/version.properties
View file
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
#Tue Jan 31 10:40:01 UTC 2017
VERSION=2.4.0-SNAPSHOT
#Thu Feb 09 15:32:51 CET 2017
VERSION=2.5.2-SNAPSHOT

0 comments on commit c692838

Please sign in to comment.