Fixes in Http steps

* Fix in `HttpCrawl` (`Http::crawl()`) step: when a page contains a
broken link, that can't be resolved and throws an `Exception` from the
URL library, ignore the link and log a warning message.
* Minor fix for merging HTTP headers when an `Http` step gets both,
statically defined headers and headers to use from array input.
* Also CS Fixer changed a lot of empty classes and constructors.

CHANGELOG.md

@@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [1.2.2] - 2023-09-19
+### Fixed
+* Fix in `HttpCrawl` (`Http::crawl()`) step: when a page contains a broken link, that can't be resolved and throws an `Exception` from the URL library, ignore the link and log a warning message.
+* Minor fix for merging HTTP headers when an `Http` step gets both, statically defined headers and headers to use from array input.
+
 ## [1.2.1] - 2023-08-21
 ### Fixed
 * When a URL redirects, the `trackRequestEndFor()` method of the `HttpLoader`'s `Throttler` instance is called only once at the end and with the original request URL.

src/Cache/Exceptions/MissingZlibExtensionException.php

@@ -5,6 +5,4 @@
 use Exception;
 use Psr\SimpleCache\CacheException;
 
-class MissingZlibExtensionException extends Exception implements CacheException
-{
-}
+class MissingZlibExtensionException extends Exception implements CacheException {}

src/Cache/Exceptions/ReadingCacheFailedException.php

@@ -5,6 +5,4 @@
 use Exception;
 use Psr\SimpleCache\CacheException;
 
-class ReadingCacheFailedException extends Exception implements CacheException
-{
-}
+class ReadingCacheFailedException extends Exception implements CacheException {}

src/Cache/FileCache.php

@@ -16,8 +16,7 @@ class FileCache implements CacheInterface
 
     public function __construct(
         protected readonly string $basePath,
-    ) {
-    }
+    ) {}
 
     public function useCompression(): static
     {

src/Exceptions/UnknownLoaderKeyException.php

@@ -4,6 +4,4 @@
 
 use Exception;
 
-class UnknownLoaderKeyException extends Exception
-{
-}
+class UnknownLoaderKeyException extends Exception {}

src/HttpCrawler/AnonymousHttpCrawlerBuilder.php

@@ -9,9 +9,7 @@
 
 class AnonymousHttpCrawlerBuilder
 {
-    public function __construct()
-    {
-    }
+    public function __construct() {}
 
     public function withBotUserAgent(string $productToken): HttpCrawler
     {

src/Input.php

@@ -2,6 +2,4 @@
 
 namespace Crwlr\Crawler;
 
-class Input extends Io
-{
-}
+class Input extends Io {}

src/Loader/AddLoadersToStepAction.php

@@ -11,9 +11,7 @@ class AddLoadersToStepAction
      * @param LoaderInterface|array<string, LoaderInterface> $loaders
      * @param StepInterface $step
      */
-    public function __construct(protected LoaderInterface|array $loaders, protected StepInterface $step)
-    {
-    }
+    public function __construct(protected LoaderInterface|array $loaders, protected StepInterface $step) {}
 
     /**
      * @return void

src/Loader/Http/Cookies/Date.php

@@ -10,9 +10,7 @@ class Date
 {
     protected ?DateTime $dateTime = null;
 
-    public function __construct(protected readonly string $httpDateString)
-    {
-    }
+    public function __construct(protected readonly string $httpDateString) {}
 
     /**
      * @throws InvalidArgumentException

src/Loader/Http/Cookies/Exceptions/InvalidCookieException.php

@@ -4,6 +4,4 @@
 
 use Exception;
 
-class InvalidCookieException extends Exception
-{
-}
+class InvalidCookieException extends Exception {}

src/Loader/Http/Exceptions/LoadingException.php

@@ -4,6 +4,4 @@
 
 use Exception;
 
-class LoadingException extends Exception
-{
-}
+class LoadingException extends Exception {}

src/Loader/Http/Politeness/RetryErrorResponseHandler.php

@@ -27,8 +27,7 @@ public function __construct(
         protected int $retries = 2,
         protected array $wait = [10, 60],
         protected int $maxWait = 60,
-    ) {
-    }
+    ) {}
 
     public function shouldWait(RespondedRequest $respondedRequest): bool
     {

src/Loader/Http/Politeness/TimingUnits/Microseconds.php

@@ -5,6 +5,4 @@
 /**
  * @deprecated Will be removed in v2.0.0. Use the Microseconds class from the crwlr/utils package directly instead.
  */
-final class Microseconds extends \Crwlr\Utils\Microseconds
-{
-}
+final class Microseconds extends \Crwlr\Utils\Microseconds {}

src/Loader/Http/Politeness/TimingUnits/MultipleOf.php

@@ -6,9 +6,7 @@
 
 class MultipleOf
 {
-    public function __construct(public readonly float $factor)
-    {
-    }
+    public function __construct(public readonly float $factor) {}
 
     public function calc(Microseconds $microseconds): Microseconds
     {

src/Loader/Loader.php

@@ -81,16 +81,12 @@ protected function isAllowedToBeLoaded(UriInterface $uri, bool $throwsException
     /**
      * Can be implemented in a child class to track how long a request waited for its response.
      */
-    protected function trackRequestStart(?float $microtime = null): void
-    {
-    }
+    protected function trackRequestStart(?float $microtime = null): void {}
 
     /**
      * Can be implemented in a child class to track how long a request waited for its response.
      */
-    protected function trackRequestEnd(?float $microtime = null): void
-    {
-    }
+    protected function trackRequestEnd(?float $microtime = null): void {}
 
     protected function callHook(string $hook, mixed ...$arguments): void
     {

src/Output.php

@@ -2,6 +2,4 @@
 
 namespace Crwlr\Crawler;
 
-class Output extends Io
-{
-}
+class Output extends Io {}

src/Steps/Csv.php

@@ -19,9 +19,7 @@ class Csv extends Step
     /**
      * @param array<string|null> $columnMapping
      */
-    public function __construct(protected array $columnMapping = [], protected bool $skipFirstLine = false)
-    {
-    }
+    public function __construct(protected array $columnMapping = [], protected bool $skipFirstLine = false) {}
 
     /**
      * @param array<string|null> $columnMapping

src/Steps/Filters/ClosureFilter.php

@@ -9,8 +9,7 @@ class ClosureFilter extends Filter
 {
     public function __construct(
         protected readonly Closure $closure,
-    ) {
-    }
+    ) {}
 
     /**
      * @throws Exception

src/Steps/Filters/ComparisonFilter.php

@@ -10,8 +10,7 @@ class ComparisonFilter extends Filter
     public function __construct(
         protected readonly ComparisonFilterRule $filterRule,
         protected readonly mixed $compareTo,
-    ) {
-    }
+    ) {}
 
     /**
      * @throws Exception

src/Steps/Filters/NegatedFilter.php

@@ -4,9 +4,7 @@
 
 final class NegatedFilter implements FilterInterface
 {
-    public function __construct(private readonly FilterInterface $filter)
-    {
-    }
+    public function __construct(private readonly FilterInterface $filter) {}
 
     public function useKey(string $key): static
     {

src/Steps/Filters/StringFilter.php

@@ -10,8 +10,7 @@ class StringFilter extends Filter
     public function __construct(
         protected readonly StringFilterRule $filterRule,
         protected readonly string $filterString,
-    ) {
-    }
+    ) {}
 
     /**
      * @throws Exception

src/Steps/Filters/StringLengthFilter.php

@@ -10,8 +10,7 @@ class StringLengthFilter extends Filter
     public function __construct(
         protected readonly StringLengthFilterRule $filterRule,
         protected readonly int $compareToLength,
-    ) {
-    }
+    ) {}
 
     /**
      * @throws Exception

src/Steps/Filters/UrlFilter.php

@@ -7,9 +7,7 @@
 
 class UrlFilter extends Filter
 {
-    public function __construct(protected readonly UrlFilterRule $filterRule, protected readonly string $filterString)
-    {
-    }
+    public function __construct(protected readonly UrlFilterRule $filterRule, protected readonly string $filterString) {}
 
     /**
      * @throws Exception

src/Steps/Html/DomQuery.php

@@ -31,8 +31,7 @@ abstract class DomQuery implements DomQueryInterface
 
     public function __construct(
         public readonly string $query
-    ) {
-    }
+    ) {}
 
     /**
      * When there is a <base> tag with a href attribute in an HTML document all links in the document must be resolved

src/Steps/Html/GetLink.php

@@ -32,9 +32,7 @@ class GetLink extends Step
 
     protected bool $withFragment = true;
 
-    public function __construct(protected ?string $selector = null)
-    {
-    }
+    public function __construct(protected ?string $selector = null) {}
 
     public static function isSpecialNonHttpLink(Crawler $linkElement): bool
     {

src/Steps/Json.php

@@ -12,9 +12,7 @@ class Json extends Step
     /**
      * @param mixed[] $propertyMapping
      */
-    final public function __construct(protected array $propertyMapping = [], protected ?string $each = null)
-    {
-    }
+    final public function __construct(protected array $propertyMapping = [], protected ?string $each = null) {}
 
     /**
      * @param mixed[] $propertyMapping

src/Steps/Loading/Http.php

@@ -7,6 +7,7 @@
 use Crwlr\Crawler\Steps\Loading\Http\Paginate;
 use Crwlr\Crawler\Steps\Loading\Http\Paginator;
 use Crwlr\Crawler\Steps\Loading\Http\PaginatorInterface;
+use Crwlr\Crawler\Utils\HttpHeaders;
 use Exception;
 use Generator;
 use GuzzleHttp\Psr7\Request;
@@ -51,8 +52,7 @@ public function __construct(
         protected readonly array $headers = [],
         protected readonly string|StreamInterface|null $body = null,
         protected readonly string $httpVersion = '1.1',
-    ) {
-    }
+    ) {}
 
     /**
      * @param array|(string|string[])[] $headers
@@ -288,7 +288,7 @@ protected function getRequestFromInputUri(UriInterface $uri): RequestInterface
     {
         $body = $this->inputBody ?? $this->body;
 
-        $headers = $this->inputHeaders ? $this->mergeHeaders() : $this->headers;
+        $headers = $this->mergeHeaders();
 
         return new Request($this->method, $uri, $headers, $body, $this->httpVersion);
     }
@@ -398,68 +398,20 @@ protected function addToInputHeadersFromInput(mixed $input, string $inputKey, st
             return;
         }
 
-        $this->inputHeaders[$headerName] = $this->addValuesToHeaderArray(
-            is_array($this->inputHeaders[$headerName]) ? $this->inputHeaders[$headerName] : [],
-            is_array($inputValue) ? $inputValue : [$inputValue],
-        );
+        $this->inputHeaders = HttpHeaders::addTo(HttpHeaders::normalize($this->inputHeaders), $headerName, $inputValue);
     }
 
     /**
      * @return array<string, string[]>
      */
     protected function mergeHeaders(): array
     {
-        $headers = $this->normalizeHeaders($this->headers);
+        $headers = HttpHeaders::normalize($this->headers);
 
         if (is_array($this->inputHeaders)) {
-            foreach ($this->inputHeaders as $headerName => $value) {
-                if (!array_key_exists($headerName, $headers)) {
-                    $headers[$headerName] = is_array($value) ? $value : [$value];
-
-                    continue;
-                }
-
-                if (!in_array($value, $headers[$headerName], true)) {
-                    if (is_array($value)) {
-                        $headers[$headerName] = $this->addValuesToHeaderArray($headers[$headerName], $value);
-                    } elseif (!in_array($value, $headers[$headerName], true)) {
-                        $headers[$headerName][] = $value;
-                    }
-                } else {
-                    $headers[$headerName] = [$headers[$headerName], $value];
-                }
-            }
-        }
-
-        return $headers;
-    }
-
-    /**
-     * @param array<string, string|string[]> $headers
-     * @return array<string, string[]>
-     */
-    protected function normalizeHeaders(array $headers): array
-    {
-        $normalized = [];
-
-        foreach ($headers as $headerName => $value) {
-            $normalized[$headerName] = is_array($value) ? $value : [$value];
-        }
+            $inputHeaders = HttpHeaders::normalize($this->inputHeaders);
 
-        return $normalized;
-    }
-
-    /**
-     * @param string[] $headers
-     * @param string[] $values
-     * @return string[]
-     */
-    protected function addValuesToHeaderArray(array $headers, array $values): array
-    {
-        foreach ($values as $value) {
-            if (!in_array($value, $headers, true)) {
-                $headers[] = $value;
-            }
+            $headers = HttpHeaders::merge($headers, $inputHeaders);
         }
 
         return $headers;

src/Steps/Loading/Http/Paginators/AbstractPaginator.php

@@ -9,9 +9,7 @@
 
 abstract class AbstractPaginator implements PaginatorInterface
 {
-    public function __construct(protected int $maxPages = Paginator::MAX_PAGES_DEFAULT)
-    {
-    }
+    public function __construct(protected int $maxPages = Paginator::MAX_PAGES_DEFAULT) {}
 
     public function prepareRequest(
         RequestInterface $request,

src/Steps/Loading/HttpCrawl.php

@@ -12,6 +12,7 @@
 use Generator;
 use Psr\Http\Message\UriInterface;
 use Symfony\Component\DomCrawler\Crawler;
+use Throwable;
 
 class HttpCrawl extends Http
 {
@@ -295,7 +296,13 @@ protected function getUrlsFromHtmlDocument(Document $document): array
                 continue;
             }
 
-            $url = $this->handleUrlFragment($document->baseUrl()->resolve($linkElement->attr('href') ?? ''));
+            try {
+                $url = $this->handleUrlFragment($document->baseUrl()->resolve($linkElement->attr('href') ?? ''));
+            } catch (Throwable) {
+                $this->logger?->warning('Failed to resolve a link with href: ' . $linkElement->attr('href'));
+
+                continue;
+            }
 
             if (!$this->isOnSameHostOrDomain($url)) {
                 continue;