diff --git a/.gitignore b/.gitignore index a1c2a23..8baed0d 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,10 @@ # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml hs_err_pid* + +.idea* +.gradle* +build* +build/* +gradle + diff --git a/README.md b/README.md new file mode 100644 index 0000000..5c0f911 --- /dev/null +++ b/README.md @@ -0,0 +1,20 @@ +# Thread Wars: Войны клонов на страницах Wiki + +### Чат книжного клуба .rar +https://t.me/point_rar_chat + +### Правила +Описаны тут: https://telegra.ph/WikiGame-10-07 + +### Что тут есть +В исходниках Java, в папке `repository`, лежит интерфейс `WikiGame` -- его нужно реализовать. + +Рядом с этим интерфейсом лежат разные реализации этого интерфейса -- начиная от `Executors`, и заканчивая алгоритмическим решением на `Loom`. + +В исходниках для Kotlin лежит реализация с корутинами. + +### Как работать +Нужно форкнуть себе репозиторий, написать решение, и сделать PR в `main`-ветку. + +### Полезное +Дампы ру-википедии для MySQL: https://dumps.wikimedia.org/ruwiki/latest/ diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 0000000..4a80778 --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,64 @@ +plugins { + id("java") + kotlin("jvm") version "1.9.10" + kotlin("plugin.serialization") version "1.9.10" +} + +group = "point.rar" +version = "1.0-SNAPSHOT" + +repositories { + mavenCentral() +} + +dependencies { + testImplementation(platform("org.junit:junit-bom:5.9.1")) + testImplementation("org.junit.jupiter:junit-jupiter") + implementation(kotlin("stdlib-jdk8")) + + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:1.7.3") + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-jdk9:1.7.3") + + val kotlinx_serialization_json_version = "1.4.1" + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:$kotlinx_serialization_json_version") + + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.14.2") + implementation("org.apache.httpcomponents:httpclient:4.5.13") + + val ktor_version = "2.2.3" + implementation("io.ktor:ktor-client-core:$ktor_version") + implementation("io.ktor:ktor-client-cio:$ktor_version") + implementation("io.ktor:ktor-client-content-negotiation:$ktor_version") + implementation("io.ktor:ktor-serialization-kotlinx-json:$ktor_version") + + val slf4j_version = "2.0.6" + implementation("org.slf4j:slf4j-api:$slf4j_version") + implementation("org.slf4j:slf4j-simple:$slf4j_version") + + val resilience4jVersion = "2.1.0" + implementation("io.github.resilience4j:resilience4j-kotlin:${resilience4jVersion}") + implementation("io.github.resilience4j:resilience4j-retry:${resilience4jVersion}") + implementation("io.github.resilience4j:resilience4j-timelimiter:${resilience4jVersion}") + implementation("io.github.resilience4j:resilience4j-ratelimiter:${resilience4jVersion}") + + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.14.2") + implementation("org.apache.httpcomponents:httpclient:4.5.13") + implementation("io.github.resilience4j:resilience4j-timelimiter:2.1.0") + implementation("io.github.resilience4j:resilience4j-ratelimiter:2.1.0") + + implementation("io.projectreactor:reactor-core:3.5.10") + + // mysql + val jasyncVersion = "2.2.0" + implementation("com.github.jasync-sql:jasync-mysql:${jasyncVersion}") +} + +tasks.test { + useJUnitPlatform() +} +tasks.withType { + options.compilerArgs.addAll(listOf("--enable-preview", "--add-modules", "jdk.incubator.concurrent")) +} +kotlin { + jvmToolchain(19) +} \ No newline at end of file diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..fcb6fca --- /dev/null +++ b/gradlew @@ -0,0 +1,248 @@ +#!/bin/sh + +# +# Copyright © 2015-2021 the original authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +############################################################################## +# +# Gradle start up script for POSIX generated by Gradle. +# +# Important for running: +# +# (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is +# noncompliant, but you have some other compliant shell such as ksh or +# bash, then to run this script, type that shell name before the whole +# command line, like: +# +# ksh Gradle +# +# Busybox and similar reduced shells will NOT work, because this script +# requires all of these POSIX shell features: +# * functions; +# * expansions «$var», «${var}», «${var:-default}», «${var+SET}», +# «${var#prefix}», «${var%suffix}», and «$( cmd )»; +# * compound commands having a testable exit status, especially «case»; +# * various built-in commands including «command», «set», and «ulimit». +# +# Important for patching: +# +# (2) This script targets any POSIX shell, so it avoids extensions provided +# by Bash, Ksh, etc; in particular arrays are avoided. +# +# The "traditional" practice of packing multiple parameters into a +# space-separated string is a well documented source of bugs and security +# problems, so this is (mostly) avoided, by progressively accumulating +# options in "$@", and eventually passing that to Java. +# +# Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, +# and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; +# see the in-line comments for details. +# +# There are tweaks for specific operating systems such as AIX, CygWin, +# Darwin, MinGW, and NonStop. +# +# (3) This script is generated from the Groovy template +# https://github.com/gradle/gradle/blob/HEAD/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# within the Gradle project. +# +# You can find Gradle at https://github.com/gradle/gradle/. +# +############################################################################## + +# Attempt to set APP_HOME + +# Resolve links: $0 may be a link +app_path=$0 + +# Need this for daisy-chained symlinks. +while + APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path + [ -h "$app_path" ] +do + ls=$( ls -ld "$app_path" ) + link=${ls#*' -> '} + case $link in #( + /*) app_path=$link ;; #( + *) app_path=$APP_HOME$link ;; + esac +done + +# This is normally unused +# shellcheck disable=SC2034 +APP_BASE_NAME=${0##*/} +APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD=maximum + +warn () { + echo "$*" +} >&2 + +die () { + echo + echo "$*" + echo + exit 1 +} >&2 + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "$( uname )" in #( + CYGWIN* ) cygwin=true ;; #( + Darwin* ) darwin=true ;; #( + MSYS* | MINGW* ) msys=true ;; #( + NONSTOP* ) nonstop=true ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD=$JAVA_HOME/jre/sh/java + else + JAVACMD=$JAVA_HOME/bin/java + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD=java + if ! command -v java >/dev/null 2>&1 + then + die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +fi + +# Increase the maximum file descriptors if we can. +if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then + case $MAX_FD in #( + max*) + # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 + MAX_FD=$( ulimit -H -n ) || + warn "Could not query maximum file descriptor limit" + esac + case $MAX_FD in #( + '' | soft) :;; #( + *) + # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked. + # shellcheck disable=SC3045 + ulimit -n "$MAX_FD" || + warn "Could not set maximum file descriptor limit to $MAX_FD" + esac +fi + +# Collect all arguments for the java command, stacking in reverse order: +# * args from the command line +# * the main class name +# * -classpath +# * -D...appname settings +# * --module-path (only if needed) +# * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. + +# For Cygwin or MSYS, switch paths to Windows format before running java +if "$cygwin" || "$msys" ; then + APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) + CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) + + JAVACMD=$( cygpath --unix "$JAVACMD" ) + + # Now convert the arguments - kludge to limit ourselves to /bin/sh + for arg do + if + case $arg in #( + -*) false ;; # don't mess with options #( + /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath + [ -e "$t" ] ;; #( + *) false ;; + esac + then + arg=$( cygpath --path --ignore --mixed "$arg" ) + fi + # Roll the args list around exactly as many times as the number of + # args, so each arg winds up back in the position where it started, but + # possibly modified. + # + # NB: a `for` loop captures its iteration list before it begins, so + # changing the positional parameters here affects neither the number of + # iterations, nor the values presented in `arg`. + shift # remove old arg + set -- "$@" "$arg" # push replacement arg + done +fi + + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' + +# Collect all arguments for the java command; +# * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of +# shell script including quotes and variable substitutions, so put them in +# double quotes to make sure that they get re-expanded; and +# * put everything else in single quotes, so that it's not re-expanded. + +set -- \ + "-Dorg.gradle.appname=$APP_BASE_NAME" \ + -classpath "$CLASSPATH" \ + org.gradle.wrapper.GradleWrapperMain \ + "$@" + +# Stop when "xargs" is not available. +if ! command -v xargs >/dev/null 2>&1 +then + die "xargs is not available" +fi + +# Use "xargs" to parse quoted args. +# +# With -n1 it outputs one arg per line, with the quotes and backslashes removed. +# +# In Bash we could simply go: +# +# readarray ARGS < <( xargs -n1 <<<"$var" ) && +# set -- "${ARGS[@]}" "$@" +# +# but POSIX shell has neither arrays nor command substitution, so instead we +# post-process each arg (as a line of input to sed) to backslash-escape any +# character that might be a shell metacharacter, then use eval to reverse +# that process (while maintaining the separation between arguments), and wrap +# the whole thing up as a single "set" statement. +# +# This will of course break if any of these variables contains a newline or +# an unmatched quote. +# + +eval "set -- $( + printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | + xargs -n1 | + sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | + tr '\n' ' ' + )" '"$@"' + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..6689b85 --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,92 @@ +@rem +@rem Copyright 2015 the original author or authors. +@rem +@rem Licensed under the Apache License, Version 2.0 (the "License"); +@rem you may not use this file except in compliance with the License. +@rem You may obtain a copy of the License at +@rem +@rem https://www.apache.org/licenses/LICENSE-2.0 +@rem +@rem Unless required by applicable law or agreed to in writing, software +@rem distributed under the License is distributed on an "AS IS" BASIS, +@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +@rem See the License for the specific language governing permissions and +@rem limitations under the License. +@rem + +@if "%DEBUG%"=="" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%"=="" set DIRNAME=. +@rem This is normally unused +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Resolve any "." and ".." in APP_HOME to make it shorter. +for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if %ERRORLEVEL% equ 0 goto execute + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto execute + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* + +:end +@rem End local scope for the variables with windows NT shell +if %ERRORLEVEL% equ 0 goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +set EXIT_CODE=%ERRORLEVEL% +if %EXIT_CODE% equ 0 set EXIT_CODE=1 +if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% +exit /b %EXIT_CODE% + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 0000000..537a11c --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1,8 @@ +plugins { + id("org.gradle.toolchains.foojay-resolver-convention") version "0.5.0" +} +rootProject.name = "Thread-Wars-WikiGame" +include("coroutines") +include("executor") +include("completable-future") +include("algo") diff --git a/src/main/java/rar/java/Main.java b/src/main/java/rar/java/Main.java new file mode 100644 index 0000000..b06856a --- /dev/null +++ b/src/main/java/rar/java/Main.java @@ -0,0 +1,23 @@ +package rar.java; + +import rar.java.wiki.data.repository.WikiRepositoryImpl; +import rar.java.wiki.data.source.WikiMySqlDataSourceImpl; +import rar.java.wiki.data.source.WikiDataSource; +import rar.java.wiki.domain.repository.WikiRepository; +import rar.java.repository.*; + +public class Main { + public static void main(String[] args) { + WikiDataSource wikiDataSource = new WikiMySqlDataSourceImpl(); + WikiRepository wikiRepository = new WikiRepositoryImpl(wikiDataSource); + + WikiGame wikiGame = new ReactorAlgoWikiGameImpl(wikiRepository); + + long startTime = System.currentTimeMillis(); + var path = wikiGame.play("Охотники_за_привидениями", "Пуджа", 6); + long endTime = System.currentTimeMillis(); + + System.out.println(path); + System.out.println("Total execution time: " + (endTime - startTime) + "ms"); + } +} \ No newline at end of file diff --git a/src/main/java/rar/java/repository/LoomAlgoImpl.java b/src/main/java/rar/java/repository/LoomAlgoImpl.java new file mode 100644 index 0000000..46e8a11 --- /dev/null +++ b/src/main/java/rar/java/repository/LoomAlgoImpl.java @@ -0,0 +1,171 @@ +package rar.java.repository; + +import jdk.incubator.concurrent.StructuredTaskScope; +import rar.java.wiki.domain.repository.WikiRepository; +import rar.kotlin.model.BackwardPage; +import rar.kotlin.model.ForwardPage; + +import java.util.*; +import java.util.concurrent.*; + +public class LoomAlgoImpl implements WikiGame { + + private record PairPages(ForwardPage forwardPage, BackwardPage backwardPage) { + } + + private final WikiRepository wikiRepository; + + public LoomAlgoImpl(WikiRepository wikiRepository) { + this.wikiRepository = wikiRepository; + } + + @Override + public List play( + String startPageTitle, + String endPageTitle, + int maxDepth + ) { + var visitedForwardPages = new ConcurrentHashMap(); + var visitedBackwardPages = new ConcurrentHashMap(); + + var startForwardPage = new ForwardPage(startPageTitle, null); + var endBackwardPage = new BackwardPage(endPageTitle, null); + + try (var scope = new StructuredTaskScope.ShutdownOnSuccess()) { + scope.fork(() -> processPageForward( + startForwardPage, + 0, + maxDepth, + visitedForwardPages, + visitedBackwardPages + )); + + scope.fork(() -> processPageBackward( + endBackwardPage, + 0, + maxDepth, + visitedForwardPages, + visitedBackwardPages + )); + + scope.join(); + + var pairPagesResult = scope.result(); + + return getFinalPathFromForwardAndBackward(pairPagesResult.forwardPage, pairPagesResult.backwardPage); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + private PairPages processPageForward( + ForwardPage page, + int curDepth, + int maxDepth, + ConcurrentMap visitedForwardPages, + ConcurrentMap visitedBackwardPages + ) { + if (visitedForwardPages.putIfAbsent(page.getTitle(), page) != null) { + throw new RuntimeException("Already visited"); + } + + var backwardPage = visitedBackwardPages.get(page.getTitle()); + if (backwardPage != null) { + return new PairPages(page, backwardPage); + } + + if (curDepth == maxDepth) { + throw new RuntimeException("Depth reached"); + } + + var links = wikiRepository.getLinksByTitle(page.getTitle()); + + try (var scope = new StructuredTaskScope.ShutdownOnSuccess()) { + links.forEach((link) -> { + scope.fork(() -> processPageForward( + new ForwardPage(link, page), + curDepth + 1, + maxDepth, + visitedForwardPages, + visitedBackwardPages + ) + ); + }); + + scope.join(); + + return scope.result(); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + private PairPages processPageBackward( + BackwardPage page, + int curDepth, + int maxDepth, + ConcurrentMap visitedForwardPages, + ConcurrentMap visitedBackwardPages + ) { + if (visitedBackwardPages.putIfAbsent(page.getTitle(), page) != null) { + throw new RuntimeException("Already visited"); + } + + var forwardPage = visitedForwardPages.get(page.getTitle()); + if (forwardPage != null) { + return new PairPages(forwardPage, page); + } + + if (curDepth == maxDepth) { + throw new RuntimeException("Depth reached"); + } + + var backlinks = wikiRepository.getBacklinksByTitle(page.getTitle()); + + try (var scope = new StructuredTaskScope.ShutdownOnSuccess()) { + backlinks.forEach((link) -> { + scope.fork(() -> processPageBackward( + new BackwardPage(link, page), + curDepth + 1, + maxDepth, + visitedForwardPages, + visitedBackwardPages + )); + }); + + scope.join(); + + return scope.result(); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + private List getFinalPathFromForwardAndBackward( + ForwardPage forwardPage, + BackwardPage backwardPage + ) { + var path = new ArrayList(); + + var forwardPages = new ArrayList(); + var curFwdPage = forwardPage; + while (curFwdPage != null) { + forwardPages.add(curFwdPage); + curFwdPage = curFwdPage.getParentPage(); + } + + Collections.reverse(forwardPages); + for (var fwdPg : forwardPages) { + path.add(fwdPg.getTitle()); + } + + var curBwdPage = backwardPage.getChildPage(); + while (curBwdPage != null) { + path.add(curBwdPage.getTitle()); + curBwdPage = curBwdPage.getChildPage(); + } + + return path; + } + +} diff --git a/src/main/java/rar/java/repository/LoomImpl.java b/src/main/java/rar/java/repository/LoomImpl.java new file mode 100644 index 0000000..9f8c0a5 --- /dev/null +++ b/src/main/java/rar/java/repository/LoomImpl.java @@ -0,0 +1,78 @@ +package rar.java.repository; + +import jdk.incubator.concurrent.StructuredTaskScope; +import rar.java.wiki.domain.repository.WikiRepository; +import rar.kotlin.model.Page; + +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; + +public class LoomImpl implements WikiGame { + + private final WikiRepository wikiRepository; + + public LoomImpl(WikiRepository wikiRepository) { + this.wikiRepository = wikiRepository; + } + + @Override + public List play(String startPageTitle, String endPageTitle, int maxDepth) { + var visitedPages = new ConcurrentHashMap(); + + var startPage = new Page(startPageTitle, null); + + Page resultPage = processPage(startPage, endPageTitle, 0, maxDepth, visitedPages); + + var path = new ArrayList(); + + var curPg = resultPage; + do { + path.add(curPg.getTitle()); + curPg = curPg.getParentPage(); + } while (curPg != null); + + Collections.reverse(path); + return path; + } + + private Page processPage( + Page page, + String endPageTitle, + int curDepth, + int maxDepth, + Map visitedPages) { + + if (visitedPages.putIfAbsent(page.getTitle(), true) != null) { + throw new RuntimeException("Already visited"); + } + + if (page.getTitle().equals(endPageTitle)) { + return page; + } + + if (curDepth == maxDepth) { + throw new RuntimeException("Depth reached"); + } + + var links = wikiRepository.getLinksByTitle(page.getTitle()); + + try (var scope = new StructuredTaskScope.ShutdownOnSuccess()) { + links.forEach((link) -> { + scope.fork(() -> processPage( + new Page(link, page), + endPageTitle, + curDepth + 1, + maxDepth, + visitedPages) + ); + }); + + scope.join(); + + return scope.result(); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/rar/java/repository/ReactorAlgoWikiGameImpl.java b/src/main/java/rar/java/repository/ReactorAlgoWikiGameImpl.java new file mode 100644 index 0000000..4e4e450 --- /dev/null +++ b/src/main/java/rar/java/repository/ReactorAlgoWikiGameImpl.java @@ -0,0 +1,100 @@ +package rar.java.repository; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.ConcurrentHashMap; +import java.util.function.Function; + +import rar.java.wiki.domain.repository.WikiRepository; +import rar.kotlin.model.BackwardPage; +import rar.kotlin.model.ForwardPage; +import reactor.core.publisher.Flux; +import reactor.core.publisher.Mono; +import reactor.core.scheduler.Schedulers; + +public class ReactorAlgoWikiGameImpl implements WikiGame { + + + private record PairPages(ForwardPage forwardPage, BackwardPage backwardPage) { + } + + + private final WikiRepository wikiRepository; + + public ReactorAlgoWikiGameImpl(WikiRepository wikiRepository) { + this.wikiRepository = wikiRepository; + } + + @Override + public List play(String startPageTitle, String endPageTitle, int maxDepth) { + var visitedForwardPages = new ConcurrentHashMap(); + var visitedBackwardPages = new ConcurrentHashMap(); + + var startForwardPage = new ForwardPage(startPageTitle, null); + var endBackwardPage = new BackwardPage(endPageTitle, null); + + PairPages pairPagesResult = Flux.firstWithValue( + Mono.just(startForwardPage) + .expand(page -> getLinks(page) + .map(link -> new ForwardPage(link, page)) + .filter(page2 -> visitedForwardPages.putIfAbsent(page2.getTitle(), page2) == null) + ) + .filter(page -> visitedBackwardPages.containsKey(page.getTitle())) + .map(page -> new PairPages(page, visitedBackwardPages.get(page.getTitle()))), + + + Mono.just(endBackwardPage) + .expand(page -> getBackwardLinks(page) + .map(link -> new BackwardPage(link, page)) + .filter(page2 -> visitedBackwardPages.putIfAbsent(page2.getTitle(), page2) == null) + ) + .filter(page -> visitedForwardPages.containsKey(page.getTitle())) + .map(page -> new PairPages(visitedForwardPages.get(page.getTitle()), page)) + ) + .blockFirst(); + + return getFinalPathFromForwardAndBackward(pairPagesResult.forwardPage, pairPagesResult.backwardPage); + } + + private Flux getLinks(ForwardPage page) { + return Mono.fromCallable(() -> wikiRepository.getLinksByTitle(page.getTitle())) + .subscribeOn(Schedulers.boundedElastic()) + .flatMapIterable(Function.identity()); + } + + private Flux getBackwardLinks(BackwardPage page) { + return Mono.fromCallable(() -> wikiRepository.getBacklinksByTitle(page.getTitle())) + .subscribeOn(Schedulers.boundedElastic()) + .flatMapIterable(Function.identity()); + } + + private List getFinalPathFromForwardAndBackward( + ForwardPage forwardPage, + BackwardPage backwardPage + ) { + var path = new ArrayList(); + + var forwardPages = new ArrayList(); + var curFwdPage = forwardPage; + while (curFwdPage != null) { + forwardPages.add(curFwdPage); + curFwdPage = curFwdPage.getParentPage(); + } + + Collections.reverse(forwardPages); + for (var fwdPg : forwardPages) { + path.add(fwdPg.getTitle()); + } + + var curBwdPage = backwardPage.getChildPage(); + while (curBwdPage != null) { + path.add(curBwdPage.getTitle()); + curBwdPage = curBwdPage.getChildPage(); + } + + return path; + } + + +} diff --git a/src/main/java/rar/java/repository/WikiGame.java b/src/main/java/rar/java/repository/WikiGame.java new file mode 100644 index 0000000..fe8e365 --- /dev/null +++ b/src/main/java/rar/java/repository/WikiGame.java @@ -0,0 +1,7 @@ +package rar.java.repository; + +import java.util.List; + +public interface WikiGame { + List play(String startPageTitle, String endPageTitle, int maxDepth); +} diff --git a/src/main/java/rar/java/repository/WikiGameExecutorImpl.java b/src/main/java/rar/java/repository/WikiGameExecutorImpl.java new file mode 100644 index 0000000..0702e58 --- /dev/null +++ b/src/main/java/rar/java/repository/WikiGameExecutorImpl.java @@ -0,0 +1,88 @@ +package rar.java.repository; + +import org.jetbrains.annotations.NotNull; +import rar.java.wiki.data.source.WikiDataSource; +import rar.java.wiki.data.source.WikiDataSourceImpl; +import rar.kotlin.model.Page; + +import java.util.*; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; + +public class WikiGameExecutorImpl implements WikiGame { + + private static AtomicBoolean isFinished = new AtomicBoolean(false); + + private final WikiDataSource wikiDataSource = new WikiDataSourceImpl(); + + @NotNull + @Override + public List play(@NotNull String startPageTitle, @NotNull String endPageTitle, int maxDepth) { + var startPage = new Page(startPageTitle, null); + Queue rawPages = new ConcurrentLinkedQueue<>(Collections.singleton(startPage)); + Queue parsedPages = new ConcurrentLinkedQueue<>(); + Set receivedLinks = new ConcurrentSkipListSet<>(); + + var executor = Executors.newCachedThreadPool(); + + do { + executor.execute(makeSearch(rawPages, parsedPages, receivedLinks, endPageTitle)); + } while (!isFinished.get()); + executor.shutdown(); + executor.close(); + + parsedPages.add( + new Page(endPageTitle, + rawPages.stream() + .filter(p -> p.getTitle().equals(endPageTitle)) + .findAny() + .orElseThrow() + ) + ); + + return getResultPath(parsedPages, endPageTitle); + } + + public Runnable makeSearch(Queue rawPages, Queue parsedPages, Set receivedLinks, String endPageTitle) { + return () -> { + Page currentPage = rawPages.poll(); + if (currentPage != null) { + List newLinks; + try { + newLinks = wikiDataSource.getLinksByTitle(currentPage.getTitle()); + } catch (Throwable e) { + newLinks = null; + } + + if (newLinks != null) { + parsedPages.add(currentPage); + for (String link : newLinks) { + if (receivedLinks.add(link)) { + rawPages.add(new Page(link, currentPage)); + } + if (endPageTitle.equals(link)) { + isFinished.set(true); + break; + } + } + } else { + rawPages.add(currentPage); + } + } + }; + } + + private static List getResultPath(Queue parsedPages, String endPageTitle) { + var path = new ArrayList(); + var currentPage = parsedPages.stream() + .filter(p -> p.getTitle().equals(endPageTitle)) + .findAny() + .orElseThrow(); + while (currentPage.getParentPage() != null) { + currentPage = currentPage.getParentPage(); + path.add(currentPage.getTitle()); + } + Collections.reverse(path); // Reverse the order of elements in the list + return path; + } +} diff --git a/src/main/java/rar/java/repository/WikiGameFutureImpl.java b/src/main/java/rar/java/repository/WikiGameFutureImpl.java new file mode 100644 index 0000000..cc3732a --- /dev/null +++ b/src/main/java/rar/java/repository/WikiGameFutureImpl.java @@ -0,0 +1,85 @@ +package rar.java.repository; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.jetbrains.annotations.NotNull; +import rar.java.wiki.data.source.WikiDataSource; +import rar.java.wiki.data.source.WikiDataSourceImpl; +import rar.kotlin.model.Page; + +import java.util.*; +import java.util.concurrent.*; + +public class WikiGameFutureImpl implements WikiGame { + + private static final WikiDataSource WIKI_DATA_SOURCE = new WikiDataSourceImpl(); + + @NotNull + @Override + public List play(String startPageTitle, String endPageTitle, int maxDepth) { + var startedPage = new Page(startPageTitle, null); + Queue rawPages = new ConcurrentLinkedQueue<>(Collections.singleton(startedPage)); + Queue parsedPages = new ConcurrentLinkedQueue<>(); + Set parsedTitle = new ConcurrentSkipListSet<>(); + + ExecutorService executorService = Executors.newCachedThreadPool(); +// ExecutorService executorService = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors()); +// ExecutorService executorService = Executors.newVirtualThreadPerTaskExecutor(); + List> futures = new ArrayList<>(); + + do { + for (int i = 0; i < 10; i++) { + Page curPage = rawPages.poll(); + if (curPage == null) { + break; + } + CompletableFuture future = CompletableFuture.runAsync(() -> makeSearch(curPage, rawPages, parsedPages, parsedTitle), executorService); + futures.add(future); + } + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + futures.clear(); + } while (!parsedTitle.contains(endPageTitle)); + parsedPages.add( + new Page(endPageTitle, + rawPages.stream() + .filter(p -> p.getTitle().equals(endPageTitle)) + .findAny() + .orElseThrow() + ) + ); + + executorService.shutdown(); + return getResultPath(parsedPages, endPageTitle); + } + + private void makeSearch(Page curPage, Queue rawPages, Queue parsedPages, Set parsedTitle) { + List newLinks = getChildLinks(curPage.getTitle()); + parsedPages.add(curPage); + parsedTitle.addAll(newLinks); + rawPages.addAll(newLinks.stream() + .map(m -> new Page(m, curPage)) + .toList()); + } + + private List getResultPath(Queue parsedPages, String endPageTitle) { + var path = new ArrayList(); + var curPage = parsedPages.stream() + .filter(p -> p.getTitle().equals(endPageTitle)) + .findAny() + .orElseThrow(); + while (curPage.getParentPage() != null) { + curPage = curPage.getParentPage(); + path.add(curPage.getTitle()); + } + Collections.reverse(path); // Reverse the order of elements in the list + return path; + } + + + private List getChildLinks(String title) { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + return WIKI_DATA_SOURCE.getLinksByTitle(title); + } +} diff --git a/src/main/java/rar/java/repository/WikiGameFutureSimpleImpl.java b/src/main/java/rar/java/repository/WikiGameFutureSimpleImpl.java new file mode 100644 index 0000000..e48d3e8 --- /dev/null +++ b/src/main/java/rar/java/repository/WikiGameFutureSimpleImpl.java @@ -0,0 +1,136 @@ +package rar.java.repository; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.github.resilience4j.ratelimiter.RateLimiter; +import io.github.resilience4j.ratelimiter.RateLimiterConfig; +import io.github.resilience4j.ratelimiter.RateLimiterRegistry; +import org.apache.http.client.utils.URIBuilder; +import org.jetbrains.annotations.NotNull; +import rar.kotlin.model.Link; +import rar.kotlin.model.Page; +import rar.kotlin.model.WikiLinksResponse; + +import java.io.IOException; +import java.net.URI; +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.*; +import java.util.concurrent.*; + +public class WikiGameFutureSimpleImpl implements WikiGame { + private static final String URL = "https://ru.wikipedia.org/w/api.php"; + + @NotNull + @Override + public List play(String startPageTitle, String endPageTitle, int maxDepth) { + var startedPage = new Page(startPageTitle, null); + Queue rawPages = new ConcurrentLinkedQueue<>(Collections.singleton(startedPage)); + Queue parsedPages = new ConcurrentLinkedQueue<>(); + Set parsedTitle = new ConcurrentSkipListSet<>(); + List> futures = new ArrayList<>(); + + RateLimiterConfig config = RateLimiterConfig.custom() + .limitRefreshPeriod(Duration.ofMillis(1000)) + .limitForPeriod(300) + .timeoutDuration(Duration.ofMillis(25)) + .build(); + RateLimiterRegistry rateLimiterRegistry = RateLimiterRegistry.of(config); + RateLimiter rateLimiter = rateLimiterRegistry + .rateLimiter("rateLimiter", config); + + + ExecutorService executorService = Executors.newVirtualThreadPerTaskExecutor(); + do { + for (int i = 0; i < 1000; i++) { + Page curPage = rawPages.poll(); + if (curPage == null) { + break; + } + Runnable restrictedCall = RateLimiter + .decorateRunnable(rateLimiter, makeSearch(curPage, rawPages, parsedPages, parsedTitle)); + CompletableFuture future =CompletableFuture.runAsync(restrictedCall); + futures.add(future); + } + CompletableFuture.allOf(futures.toArray(new CompletableFuture[0])).join(); + futures.clear(); + } while (!parsedTitle.contains(endPageTitle)); + parsedPages.add( + new Page(endPageTitle, + rawPages.stream() + .filter(p -> p.getTitle().equals(endPageTitle)) + .findAny() + .orElseThrow() + ) + ); + + executorService.shutdown(); + return getResultPath(parsedPages, endPageTitle); + } + + private Runnable makeSearch(Page curPage, Queue rawPages, Queue parsedPages, Set parsedTitle) { + return () -> { + List newLinks = getChildLinks(curPage.getTitle()); + parsedPages.add(curPage); + parsedTitle.addAll(newLinks); + rawPages.addAll(newLinks.stream() + .map(m -> new Page(m, curPage)) + .toList()); + }; + } + + private List getResultPath(Queue parsedPages, String endPageTitle) { + var path = new ArrayList(); + var curPage = parsedPages.stream() + .filter(p -> p.getTitle().equals(endPageTitle)) + .findAny() + .orElseThrow(); + while (curPage.getParentPage() != null) { + curPage = curPage.getParentPage(); + path.add(curPage.getTitle()); + } + Collections.reverse(path); // Reverse the order of elements in the list + return path; + } + + + private static List getChildLinks(String title) { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + + String responseBody = null; + try { + URIBuilder uriBuilder = new URIBuilder(URL); + uriBuilder.addParameter("action", "query"); + uriBuilder.addParameter("prop", "links"); + uriBuilder.addParameter("pllimit", "max"); + uriBuilder.addParameter("format", "json"); + uriBuilder.addParameter("plnamespace", "titles"); + + System.out.println("Get links for: " + title); + responseBody = HttpClient.newBuilder() + .build() + .sendAsync(HttpRequest.newBuilder() + .uri(URI.create(uriBuilder.addParameter("titles", title).build().toString())) + .GET() + .build(), HttpResponse.BodyHandlers.ofString()).get().body(); + var response = objectMapper.readValue(responseBody, WikiLinksResponse.class); + + return parseLinks(response); + } catch (IOException | InterruptedException | URISyntaxException | ExecutionException e) { + throw new RuntimeException(e); + } + } + + @NotNull + private static List parseLinks(WikiLinksResponse response) { + return response.getQuery().getPages().entrySet().iterator().next().getValue().getLinks() + .stream() + .map(Link::getTitle) + .toList(); + } +} diff --git a/src/main/java/rar/java/repository/WikiGameSerialImpl.java b/src/main/java/rar/java/repository/WikiGameSerialImpl.java new file mode 100644 index 0000000..0111a90 --- /dev/null +++ b/src/main/java/rar/java/repository/WikiGameSerialImpl.java @@ -0,0 +1,56 @@ +package rar.java.repository; + +import org.jetbrains.annotations.NotNull; +import rar.java.wiki.data.source.WikiDataSource; +import rar.java.wiki.data.source.WikiDataSourceImpl; +import rar.kotlin.model.Page; + +import java.util.*; + +public class WikiGameSerialImpl implements WikiGame { + private static final String URL = "https://ru.wikipedia.org/w/api.php"; + + private final WikiDataSource wikiDataSource = new WikiDataSourceImpl(); + + @NotNull + @Override + public List play(String startPageTitle, String endPageTitle, int maxDepth) { + var startedPage = new Page(startPageTitle, null); + Queue rawPages = new LinkedList<>(Collections.singleton(startedPage)); + Queue parsedPages = new LinkedList<>(); + Set parsedTitle = new HashSet<>(); + var parentEndPage = startedPage; + do { + System.out.println("parsedPages size = " + parsedPages.size()); + System.out.println("rawPages size = " + rawPages.size()); + System.out.println("parsedTitle size = " + parsedTitle.size()); + var curPage = rawPages.poll(); + parentEndPage = curPage; + var newLinks = wikiDataSource.getLinksByTitle(curPage.getTitle()); + parsedPages.add(curPage); + parsedTitle.addAll(newLinks); + rawPages.addAll( + newLinks.stream() + .map(m -> new Page(m, curPage)) + .toList() + ); + } while (!parsedTitle.contains(endPageTitle)); + parsedPages.add(new Page(endPageTitle, parentEndPage)); + + return getResultPath(parsedPages, endPageTitle); + } + + private List getResultPath(Queue parsedPages, String endPageTitle) { + var path = new ArrayList(); + var curPage = parsedPages.stream() + .filter(p -> p.getTitle().equals(endPageTitle)) + .findAny() + .orElseThrow(); + while (curPage.getParentPage() != null) { + path.add(curPage.getTitle()); + curPage = curPage.getParentPage(); + } + path.add(curPage.getTitle()); + return path; + } +} diff --git a/src/main/java/rar/java/wiki/data/repository/WikiRepositoryImpl.java b/src/main/java/rar/java/wiki/data/repository/WikiRepositoryImpl.java new file mode 100644 index 0000000..d7c0637 --- /dev/null +++ b/src/main/java/rar/java/wiki/data/repository/WikiRepositoryImpl.java @@ -0,0 +1,26 @@ +package rar.java.wiki.data.repository; + +import rar.java.wiki.data.source.WikiDataSource; +import rar.java.wiki.domain.repository.WikiRepository; + +import java.util.List; + +public class WikiRepositoryImpl implements WikiRepository { + + private final WikiDataSource wikiDataSource; + + public WikiRepositoryImpl(WikiDataSource wikiDataSource) { + this.wikiDataSource = wikiDataSource; + } + + + @Override + public List getLinksByTitle(String title) { + return wikiDataSource.getLinksByTitle(title); + } + + @Override + public List getBacklinksByTitle(String title) { + return wikiDataSource.getBacklinksByTitle(title); + } +} diff --git a/src/main/java/rar/java/wiki/data/source/WikiDataSource.java b/src/main/java/rar/java/wiki/data/source/WikiDataSource.java new file mode 100644 index 0000000..1dcc6ce --- /dev/null +++ b/src/main/java/rar/java/wiki/data/source/WikiDataSource.java @@ -0,0 +1,8 @@ +package rar.java.wiki.data.source; + +import java.util.List; + +public interface WikiDataSource { + List getLinksByTitle(String title); + List getBacklinksByTitle(String title); +} diff --git a/src/main/java/rar/java/wiki/data/source/WikiDataSourceImpl.java b/src/main/java/rar/java/wiki/data/source/WikiDataSourceImpl.java new file mode 100644 index 0000000..3d67455 --- /dev/null +++ b/src/main/java/rar/java/wiki/data/source/WikiDataSourceImpl.java @@ -0,0 +1,114 @@ +package rar.java.wiki.data.source; + +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import io.github.resilience4j.ratelimiter.RateLimiter; +import io.github.resilience4j.ratelimiter.RateLimiterConfig; +import io.github.resilience4j.ratelimiter.RateLimiterRegistry; +import org.apache.http.client.utils.URIBuilder; +import org.jetbrains.annotations.NotNull; +import rar.kotlin.model.Link; +import rar.kotlin.model.WikiBacklinksResponse; +import rar.kotlin.model.WikiLinksResponse; + +import java.net.URISyntaxException; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.time.Duration; +import java.util.List; + +public class WikiDataSourceImpl implements WikiDataSource { + private static final String URL = "https://ru.wikipedia.org/w/api.php"; + + private static final RateLimiterConfig config = RateLimiterConfig.custom() + .limitRefreshPeriod(Duration.ofMillis(40)) + .limitForPeriod(1) + .timeoutDuration(Duration.ofDays(100000)) + .build(); + private static final RateLimiterRegistry rateLimiterRegistry = RateLimiterRegistry.of(config); + private static final RateLimiter rateLimiter = rateLimiterRegistry.rateLimiter("rate"); + private static final ObjectMapper objectMapper = new ObjectMapper() + .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); + + private static final HttpClient httpClient = HttpClient.newHttpClient(); + + @Override + public List getLinksByTitle(String title) { + try { + String responseBody = RateLimiter.decorateCheckedSupplier(rateLimiter, () -> + httpClient.send( + HttpRequest.newBuilder() + .uri(getUriBuilder(title).build()) + .GET() + .build(), + HttpResponse.BodyHandlers.ofString() + ) + .body() + ).get(); + + WikiLinksResponse response = objectMapper.readValue(responseBody, WikiLinksResponse.class); + return parseResponse(response); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @NotNull + private static URIBuilder getUriBuilder(String title) throws URISyntaxException { + URIBuilder uriBuilder = new URIBuilder(URL); + uriBuilder.addParameter("action", "query"); + uriBuilder.addParameter("prop", "links"); + uriBuilder.addParameter("pllimit", "max"); + uriBuilder.addParameter("format", "json"); + uriBuilder.addParameter("plnamespace", "titles"); + uriBuilder.addParameter("titles", title); + return uriBuilder; + } + + @NotNull + private static List parseResponse(WikiLinksResponse response) { + return response.getQuery().getPages().entrySet().iterator().next().getValue().getLinks() + .stream() + .map(Link::getTitle) + .toList(); + } + + @Override + public List getBacklinksByTitle(String title) { + try { + String responseBody = RateLimiter.decorateCheckedSupplier(rateLimiter, () -> + httpClient.send( + HttpRequest.newBuilder() + .uri(getBacklinksUriBuilder(title).build()) + .GET() + .build(), + HttpResponse.BodyHandlers.ofString() + ) + .body() + ).get(); + WikiBacklinksResponse response = objectMapper.readValue(responseBody, WikiBacklinksResponse.class); + + return parseBacklinksResponse(response); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @NotNull + private static URIBuilder getBacklinksUriBuilder(String title) throws URISyntaxException { + URIBuilder uriBuilder = new URIBuilder(URL); + uriBuilder.addParameter("action", "query"); + uriBuilder.addParameter("bltitle", title); + uriBuilder.addParameter("list", "backlinks"); + uriBuilder.addParameter("bllimit", "max"); + uriBuilder.addParameter("format", "json"); + uriBuilder.addParameter("blnamespace", "0"); + return uriBuilder; + } + + @NotNull + private static List parseBacklinksResponse(WikiBacklinksResponse response) { + return response.getQuery().getBacklinks().stream().map(Link::getTitle).toList(); + } +} diff --git a/src/main/java/rar/java/wiki/data/source/WikiMySqlDataSourceImpl.java b/src/main/java/rar/java/wiki/data/source/WikiMySqlDataSourceImpl.java new file mode 100644 index 0000000..13b9ece --- /dev/null +++ b/src/main/java/rar/java/wiki/data/source/WikiMySqlDataSourceImpl.java @@ -0,0 +1,65 @@ +package rar.java.wiki.data.source; + + +import com.github.jasync.sql.db.Connection; +import com.github.jasync.sql.db.mysql.MySQLConnectionBuilder; +import io.github.resilience4j.ratelimiter.RateLimiter; +import io.github.resilience4j.ratelimiter.RateLimiterConfig; +import io.github.resilience4j.ratelimiter.RateLimiterRegistry; + +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Objects; + +public class WikiMySqlDataSourceImpl implements WikiDataSource { + private static final RateLimiterConfig config = RateLimiterConfig.custom() + .limitRefreshPeriod(Duration.ofMillis(100)) + .limitForPeriod(1) + .timeoutDuration(Duration.ofDays(100000)) + .build(); + private static final RateLimiterRegistry rateLimiterRegistry = RateLimiterRegistry.of(config); + private static final RateLimiter rateLimiter = rateLimiterRegistry.rateLimiter("rate"); + + // Connection to MySQL DB + Connection connection = MySQLConnectionBuilder.createConnectionPool( + "jdbc:mysql://localhost:3306/wiki?user=root&password=123456789"); + + @Override + public List getLinksByTitle(String title) { + try { + var queryResult = RateLimiter.decorateCheckedSupplier(rateLimiter, () -> + connection.sendPreparedStatement( + "SELECT pl_title from pagelinks join page on page_id = pl_from where page_title = ? and pl_namespace = 0", + Collections.singletonList(title) + ).join()); + + return queryResult.get().getRows().stream() + .map(row -> row.get(0)) + .map(bytes -> new String((byte[]) bytes, StandardCharsets.UTF_8)) + .toList(); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } + + @Override + public List getBacklinksByTitle(String title) { + try { + var queryResult = RateLimiter.decorateCheckedSupplier(rateLimiter, () -> + connection.sendPreparedStatement( + "SELECT page_title from pagelinks join page on pl_from = page_id where pl_title = ? and pl_namespace = 0;", + Collections.singletonList(title) + ).join()); + + return queryResult.get().getRows().stream() + .map(row -> row.get(0)) + .filter(Objects::nonNull) + .map(bytes -> new String((byte[]) bytes, StandardCharsets.UTF_8)) + .toList(); + } catch (Throwable e) { + throw new RuntimeException(e); + } + } +} diff --git a/src/main/java/rar/java/wiki/domain/repository/WikiRepository.java b/src/main/java/rar/java/wiki/domain/repository/WikiRepository.java new file mode 100644 index 0000000..bc7a20c --- /dev/null +++ b/src/main/java/rar/java/wiki/domain/repository/WikiRepository.java @@ -0,0 +1,8 @@ +package rar.java.wiki.domain.repository; + +import java.util.List; + +public interface WikiRepository { + List getLinksByTitle(String title); + List getBacklinksByTitle(String title); +} diff --git a/src/main/kotlin/rar/kotlin/model/BackwardPage.kt b/src/main/kotlin/rar/kotlin/model/BackwardPage.kt new file mode 100644 index 0000000..ac01b2b --- /dev/null +++ b/src/main/kotlin/rar/kotlin/model/BackwardPage.kt @@ -0,0 +1,3 @@ +package rar.kotlin.model + +data class BackwardPage(val title: String, val childPage: BackwardPage?) diff --git a/src/main/kotlin/rar/kotlin/model/ForwardPage.kt b/src/main/kotlin/rar/kotlin/model/ForwardPage.kt new file mode 100644 index 0000000..f7a1abf --- /dev/null +++ b/src/main/kotlin/rar/kotlin/model/ForwardPage.kt @@ -0,0 +1,3 @@ +package rar.kotlin.model + +data class ForwardPage(val title: String, val parentPage: ForwardPage?) diff --git a/src/main/kotlin/rar/kotlin/model/Link.kt b/src/main/kotlin/rar/kotlin/model/Link.kt new file mode 100644 index 0000000..ffcb82c --- /dev/null +++ b/src/main/kotlin/rar/kotlin/model/Link.kt @@ -0,0 +1,8 @@ +package rar.kotlin.model + +import kotlinx.serialization.Serializable + +@Serializable +data class Link( + val title: String = "" +) diff --git a/src/main/kotlin/rar/kotlin/model/Page.kt b/src/main/kotlin/rar/kotlin/model/Page.kt new file mode 100644 index 0000000..908c607 --- /dev/null +++ b/src/main/kotlin/rar/kotlin/model/Page.kt @@ -0,0 +1,7 @@ +package rar.kotlin.model + +data class Page(val title: String, val parentPage: Page?) : Comparable { + override fun compareTo(other: Page): Int { + return title.compareTo(other.title) + } +} diff --git a/src/main/kotlin/rar/kotlin/model/PageLinks.kt b/src/main/kotlin/rar/kotlin/model/PageLinks.kt new file mode 100644 index 0000000..8a1b20e --- /dev/null +++ b/src/main/kotlin/rar/kotlin/model/PageLinks.kt @@ -0,0 +1,8 @@ +package rar.kotlin.model + +import kotlinx.serialization.Serializable + +@Serializable +data class PageLinks( + val links: List? = emptyList() +) diff --git a/src/main/kotlin/rar/kotlin/model/QueryBacklinks.kt b/src/main/kotlin/rar/kotlin/model/QueryBacklinks.kt new file mode 100644 index 0000000..faa89fc --- /dev/null +++ b/src/main/kotlin/rar/kotlin/model/QueryBacklinks.kt @@ -0,0 +1,8 @@ +package rar.kotlin.model + +import kotlinx.serialization.Serializable + +@Serializable +data class QueryBacklinks( + val backlinks: List = emptyList(), +) diff --git a/src/main/kotlin/rar/kotlin/model/QueryLinks.kt b/src/main/kotlin/rar/kotlin/model/QueryLinks.kt new file mode 100644 index 0000000..cfe64e3 --- /dev/null +++ b/src/main/kotlin/rar/kotlin/model/QueryLinks.kt @@ -0,0 +1,8 @@ +package rar.kotlin.model + +import kotlinx.serialization.Serializable + +@Serializable +data class QueryLinks( + val pages: Map = emptyMap() +) diff --git a/src/main/kotlin/rar/kotlin/model/WikiBacklinksResponse.kt b/src/main/kotlin/rar/kotlin/model/WikiBacklinksResponse.kt new file mode 100644 index 0000000..a95bc5d --- /dev/null +++ b/src/main/kotlin/rar/kotlin/model/WikiBacklinksResponse.kt @@ -0,0 +1,8 @@ +package rar.kotlin.model + +import kotlinx.serialization.Serializable + +@Serializable +data class WikiBacklinksResponse( + val query: QueryBacklinks = QueryBacklinks(emptyList()), +) diff --git a/src/main/kotlin/rar/kotlin/model/WikiLinksResponse.kt b/src/main/kotlin/rar/kotlin/model/WikiLinksResponse.kt new file mode 100644 index 0000000..4059298 --- /dev/null +++ b/src/main/kotlin/rar/kotlin/model/WikiLinksResponse.kt @@ -0,0 +1,8 @@ +package rar.kotlin.model + +import kotlinx.serialization.Serializable + +@Serializable +data class WikiLinksResponse( + val query: QueryLinks = QueryLinks(emptyMap()) +) \ No newline at end of file diff --git a/src/main/kotlin/rar/kotlin/repository/WikiGameCoroImpl.kt b/src/main/kotlin/rar/kotlin/repository/WikiGameCoroImpl.kt new file mode 100644 index 0000000..930a6e3 --- /dev/null +++ b/src/main/kotlin/rar/kotlin/repository/WikiGameCoroImpl.kt @@ -0,0 +1,76 @@ +package rar.kotlin.repository + +import kotlinx.coroutines.* +import kotlinx.coroutines.channels.Channel +import rar.kotlin.model.Page +import rar.java.repository.WikiGame +import rar.kotlin.wiki.WikiRemoteDataSource +import rar.kotlin.wiki.WikiRemoteDataSourceImpl +import java.util.concurrent.ConcurrentHashMap + +class WikiGameCoroImpl : WikiGame { + private val wikiRemoteDataSource: WikiRemoteDataSource = WikiRemoteDataSourceImpl() + + override fun play(startPageTitle: String, endPageTitle: String, maxDepth: Int): List = runBlocking { + val visitedPages: MutableMap = ConcurrentHashMap() + + val startPage = Page(startPageTitle, null) + + val resultPage = processPage(startPage, endPageTitle, 0, maxDepth, visitedPages) + + val path = mutableListOf() + + var curPg: Page? = resultPage + do { + path.add(curPg!!.title) + curPg = curPg.parentPage + } while (curPg != null) + + return@runBlocking path.reversed() + } + + private suspend fun processPage( + page: Page, + endPageTitle: String, + curDepth: Int, + maxDepth: Int, + visitedPages: MutableMap, + ): Page { + if (visitedPages.putIfAbsent(page.title, true) != null) { + throw RuntimeException("Already visited") + } + + if (page.title == endPageTitle) { + return page + } + + if (curDepth == maxDepth) { + throw RuntimeException("Depth reached") + } + + val links = wikiRemoteDataSource.getLinksByTitle(page.title) + + val pageChannel = Channel() + + val scope = CoroutineScope(SupervisorJob() + CoroutineExceptionHandler { _, _ -> }) + links.forEach { link -> + scope.launch { + val pageResult = processPage( + Page(link, page), + endPageTitle, + curDepth + 1, + maxDepth, + visitedPages, + ) + + pageChannel.send(pageResult) + } + } + + val resultPage = pageChannel.receive() + + scope.cancel() + + return resultPage + } +} \ No newline at end of file diff --git a/src/main/kotlin/rar/kotlin/wiki/WikiRemoteDataSource.kt b/src/main/kotlin/rar/kotlin/wiki/WikiRemoteDataSource.kt new file mode 100644 index 0000000..ba90a9a --- /dev/null +++ b/src/main/kotlin/rar/kotlin/wiki/WikiRemoteDataSource.kt @@ -0,0 +1,7 @@ +package rar.kotlin.wiki + +interface WikiRemoteDataSource { + suspend fun getLinksByTitle(title: String): List + + suspend fun getBacklinksByTitle(title: String): List +} \ No newline at end of file diff --git a/src/main/kotlin/rar/kotlin/wiki/WikiRemoteDataSourceImpl.kt b/src/main/kotlin/rar/kotlin/wiki/WikiRemoteDataSourceImpl.kt new file mode 100644 index 0000000..35324f4 --- /dev/null +++ b/src/main/kotlin/rar/kotlin/wiki/WikiRemoteDataSourceImpl.kt @@ -0,0 +1,86 @@ +package rar.kotlin.wiki + +import io.github.resilience4j.kotlin.ratelimiter.executeSuspendFunction +import io.github.resilience4j.ratelimiter.RateLimiterConfig +import io.github.resilience4j.ratelimiter.RateLimiterRegistry +import io.ktor.client.HttpClient +import io.ktor.client.call.body +import io.ktor.client.engine.cio.CIO +import io.ktor.client.plugins.contentnegotiation.ContentNegotiation +import io.ktor.client.request.get +import io.ktor.client.request.parameter +import io.ktor.serialization.kotlinx.json.json +import kotlinx.serialization.json.Json +import rar.kotlin.model.WikiBacklinksResponse +import rar.kotlin.model.WikiLinksResponse +import java.time.Duration + +class WikiRemoteDataSourceImpl : WikiRemoteDataSource { + companion object Parameter { + const val URL = "https://ru.wikipedia.org/w/api.php" + } + + private val rateLimiterConfig = RateLimiterConfig + .custom() + .limitForPeriod(1) + .limitRefreshPeriod(Duration.ofMillis(40)) + .timeoutDuration(Duration.ofDays(10000)) + .build() + + private val rateLimiterRegistry = RateLimiterRegistry.of(rateLimiterConfig) + private val rateLimiter = rateLimiterRegistry.rateLimiter("rate limiter") + + private val client: HttpClient = HttpClient(CIO) { + install(ContentNegotiation) { + json(Json { + ignoreUnknownKeys = true + }) + + } + } + + override suspend fun getLinksByTitle(title: String): List { + val response = rateLimiter.executeSuspendFunction { + client.get(URL) { + parameter("action", "query") + parameter("titles", title) + parameter("prop", "links") + parameter("pllimit", "max") + parameter("format", "json") + parameter("plnamespace", 0) + } + } + + val wikiLinksResponse: WikiLinksResponse = response.body() + + val links = wikiLinksResponse + .query + .pages + .values + .first() + .links + ?.map { it.title } ?: emptyList() + + return links + } + + override suspend fun getBacklinksByTitle(title: String): List { + val response = rateLimiter.executeSuspendFunction { + client.get(URL) { + parameter("action", "query") + parameter("bltitle", title) + parameter("list", "backlinks") + parameter("bllimit", "max") + parameter("format", "json") + parameter("blnamespace", 0) + } + } + + val wikiBacklinksResponse: WikiBacklinksResponse = response.body() + + return wikiBacklinksResponse + .query + .backlinks + .map { it.title } + } +} \ No newline at end of file