From c0e98c32c7f9887776f679f47ef471d49bd5884a Mon Sep 17 00:00:00 2001 From: Arunas Skirius Date: Wed, 12 Jul 2023 09:29:24 +0300 Subject: [PATCH] more improvements, support Nginx error logs --- config/log-viewer.php | 12 ++++ src/HttpAccessLog.php | 10 +++- ...ttpErrorLog.php => HttpApacheErrorLog.php} | 14 +++-- src/HttpLog.php | 5 ++ src/HttpLogReader.php | 22 ++++++- src/HttpNginxErrorLog.php | 59 +++++++++++++++++++ src/LogFile.php | 33 ++++++++++- src/LogViewerService.php | 30 +++++++++- tests/Pest.php | 21 ++++++- .../Fixtures/errors_nginx_dummy.log | 9 +++ ...LogTest.php => HttpApacheErrorLogTest.php} | 24 ++++++-- tests/Unit/AccessLogs/HttpLogFilesTest.php | 34 +++++++++++ tests/Unit/AccessLogs/HttpLogReaderTest.php | 4 +- .../Unit/AccessLogs/HttpNginxErrorLogTest.php | 48 +++++++++++++++ 14 files changed, 303 insertions(+), 22 deletions(-) rename src/{HttpErrorLog.php => HttpApacheErrorLog.php} (76%) create mode 100644 src/HttpNginxErrorLog.php create mode 100644 tests/Unit/AccessLogs/Fixtures/errors_nginx_dummy.log rename tests/Unit/AccessLogs/{HttpErrorLogTest.php => HttpApacheErrorLogTest.php} (65%) create mode 100644 tests/Unit/AccessLogs/HttpLogFilesTest.php create mode 100644 tests/Unit/AccessLogs/HttpNginxErrorLogTest.php diff --git a/config/log-viewer.php b/config/log-viewer.php index e3aa269d..3c9f89a0 100644 --- a/config/log-viewer.php +++ b/config/log-viewer.php @@ -149,6 +149,18 @@ // 'my_secret.log' ], + 'http_logs' => [ + 'include_files' => [ + '/var/log/httpd/*', + '/var/log/nginx/*', + // '/absolute/paths/supported', + ], + + 'exclude_files' => [ + // 'my_secret.log' + ], + ], + /* |-------------------------------------------------------------------------- | Shorter stack trace filters. diff --git a/src/HttpAccessLog.php b/src/HttpAccessLog.php index c4ce7174..346060c7 100644 --- a/src/HttpAccessLog.php +++ b/src/HttpAccessLog.php @@ -7,6 +7,8 @@ class HttpAccessLog extends HttpLog { + static string $regex = '/(\S+) (\S+) (\S+) \[(.+)\] "(\S+) (\S+) (\S+)" (\S+) (\S+) "([^"]*)" "([^"]*)"/'; + public ?string $ip; public ?string $identity; @@ -55,8 +57,7 @@ public function __construct( public function parseText(string $text = ''): array { - $regex = '/(\S+) (\S+) (\S+) \[(.+)\] "(\S+) (\S+) (\S+)" (\S+) (\S+) "([^"]*)" "([^"]*)"/'; - preg_match($regex, $text, $matches); + preg_match(self::$regex, $text, $matches); return [ 'ip' => $matches[1] ?? null, @@ -77,4 +78,9 @@ public function parseDateTime(?string $datetime): ?CarbonInterface { return $datetime ? Carbon::parse($datetime) : null; } + + public static function matches(string $text): bool + { + return preg_match(self::$regex, $text) === 1; + } } diff --git a/src/HttpErrorLog.php b/src/HttpApacheErrorLog.php similarity index 76% rename from src/HttpErrorLog.php rename to src/HttpApacheErrorLog.php index 4e338cd2..29fdca20 100644 --- a/src/HttpErrorLog.php +++ b/src/HttpApacheErrorLog.php @@ -5,8 +5,10 @@ use Carbon\CarbonInterface; use Illuminate\Support\Carbon; -class HttpErrorLog extends HttpLog +class HttpApacheErrorLog extends HttpLog { + static string $regex = '/\[(?.*?)\]\s\[(?:(?.*?):)?(?.*?)\]\s\[pid\s(?\d*)\](?:\s\[client\s(?.*?)\])?\s(?.*)/'; + public ?CarbonInterface $datetime; public ?string $module; @@ -40,15 +42,14 @@ public function __construct( public function parseText(string $text): array { - $regex = '/\[(?.*?)\]\s\[(?:(?.*?):)?(?.*?)\]\s\[pid\s(?\d*)\]\s\[client\s(?.*?)\]\s(?.*)/'; - preg_match($regex, $this->text, $matches); + preg_match(self::$regex, $this->text, $matches); return [ 'datetime' => $matches['dttm'] ?? null, 'module' => $matches['module'] ?? null, 'level' => $matches['level'] ?? null, 'pid' => $matches['pid'] ?? null, - 'client' => $matches['client'] ?? null, + 'client' => isset($matches['client']) ? ($matches['client'] ?: null) : null, 'message' => $matches['message'] ?? null, ]; } @@ -57,4 +58,9 @@ public function parseDateTime(?string $datetime): ?CarbonInterface { return $datetime ? Carbon::createFromFormat('D M d H:i:s.u Y', $datetime) : null; } + + public static function matches(string $text): bool + { + return preg_match(self::$regex, $text) === 1; + } } diff --git a/src/HttpLog.php b/src/HttpLog.php index ade15d64..d31ef892 100644 --- a/src/HttpLog.php +++ b/src/HttpLog.php @@ -11,4 +11,9 @@ public function __construct( ) { $this->text = rtrim($this->text); } + + public static function matches(string $text): bool + { + return preg_match(static::$regex, $text) === 1; + } } diff --git a/src/HttpLogReader.php b/src/HttpLogReader.php index 8cd05afa..fd3de48d 100644 --- a/src/HttpLogReader.php +++ b/src/HttpLogReader.php @@ -234,11 +234,29 @@ protected function makeLog(string $text, int $filePosition): HttpLog { return match ($this->file->type) { LogFile::TYPE_HTTP_ACCESS => new HttpAccessLog($text, $this->file->identifier, $filePosition), - LogFile::TYPE_HTTP_ERROR => new HttpErrorLog($text, $this->file->identifier, $filePosition), - default => throw new \Exception('Unknown log file type: '.$this->file->type), + LogFile::TYPE_HTTP_ERROR_APACHE => new HttpApacheErrorLog($text, $this->file->identifier, $filePosition), + LogFile::TYPE_HTTP_ERROR_NGINX => new HttpNginxErrorLog($text, $this->file->identifier, $filePosition), + default => $this->makeLogByGuessingType($text, $filePosition), }; } + protected function makeLogByGuessingType(string $text, int $filePosition): HttpLog + { + if (HttpAccessLog::matches($text)) { + return new HttpAccessLog($text, $this->file->identifier, $filePosition); + } + + if (HttpApacheErrorLog::matches($text)) { + return new HttpApacheErrorLog($text, $this->file->identifier, $filePosition); + } + + if (HttpNginxErrorLog::matches($text)) { + return new HttpNginxErrorLog($text, $this->file->identifier, $filePosition); + } + + throw new \Exception('Could not determine the log type for "'.$text.'".'); + } + public function __destruct() { $this->close(); diff --git a/src/HttpNginxErrorLog.php b/src/HttpNginxErrorLog.php new file mode 100644 index 00000000..cbd64b7b --- /dev/null +++ b/src/HttpNginxErrorLog.php @@ -0,0 +1,59 @@ +[\d+\/ :]+) \[(?P.+)\] .*?: (?P.+?)(?:, client: (?P.+?))?(?:, server: (?P.+?))?(?:, request: "?(?P.+?)"?)?(?:, host: "?(?P.+?)"?)?$/'; + + public ?CarbonInterface $datetime; + public ?string $level; + public ?string $message; + public ?string $client; + public ?string $server; + public ?string $request; + public ?string $host; + + public function __construct( + public string $text, + public ?string $fileIdentifier = null, + public ?int $filePosition = null, + ) { + parent::__construct($text, $fileIdentifier, $filePosition); + + $matches = $this->parseText($text); + + $this->datetime = $this->parseDateTime($matches['datetime'])?->tz( + config('log-viewer.timezone', config('app.timezone', 'UTC')) + ); + $this->level = $matches['level']; + $this->message = $matches['message']; + $this->client = $matches['client']; + $this->server = $matches['server']; + $this->request = $matches['request']; + $this->host = $matches['host']; + } + + public function parseText(string $text): array + { + $result = preg_match(self::$regex, $this->text, $matches); + + return [ + 'datetime' => $matches['datetime'] ?? null, + 'level' => $matches['errortype'] ?? null, + 'message' => $matches['errormessage'] ?? null, + 'client' => isset($matches['client']) ? ($matches['client'] ?: null) : null, + 'server' => $matches['server'] ?? null, + 'request' => $matches['request'] ?? null, + 'host' => $matches['host'] ?? null, + ]; + } + + public function parseDateTime(?string $datetime): ?CarbonInterface + { + return $datetime ? Carbon::createFromFormat('Y/m/d H:i:s', $datetime) : null; + } +} diff --git a/src/LogFile.php b/src/LogFile.php index 97ecd28b..2f8c6c09 100644 --- a/src/LogFile.php +++ b/src/LogFile.php @@ -3,18 +3,19 @@ namespace Opcodes\LogViewer; use Illuminate\Support\Arr; +use Illuminate\Support\Carbon; use Opcodes\LogViewer\Events\LogFileDeleted; use Opcodes\LogViewer\Exceptions\InvalidRegularExpression; +use Opcodes\LogViewer\Facades\Cache; use Opcodes\LogViewer\Utils\Utils; use Symfony\Component\HttpFoundation\BinaryFileResponse; class LogFile { const TYPE_LARAVEL = 'laravel'; - const TYPE_HTTP_ACCESS = 'http_access'; - - const TYPE_HTTP_ERROR = 'http_error'; + const TYPE_HTTP_ERROR_APACHE = 'http_error_apache'; + const TYPE_HTTP_ERROR_NGINX = 'http_error_nginx'; use Concerns\LogFile\HasMetadata; use Concerns\LogFile\CanCacheData; @@ -45,6 +46,32 @@ public function __construct(string $path, string $type = self::TYPE_LARAVEL) $this->loadMetadata(); } + public static function makeAndGuessType($filePath): self + { + $typeCacheKey = 'log-viewer::file-type-'.md5($filePath); + $type = Cache::get($typeCacheKey); + + if (isset($type)) { + return new self($filePath, $type); + } + + $reader = new HttpLogReader(new self($filePath)); + $logEntry = $reader->next(); + + $type = match (get_class($logEntry)) { + HttpAccessLog::class => LogFile::TYPE_HTTP_ACCESS, + HttpApacheErrorLog::class => LogFile::TYPE_HTTP_ERROR_APACHE, + HttpNginxErrorLog::class => LogFile::TYPE_HTTP_ERROR_NGINX, + default => null, + }; + + if (isset($type)) { + Cache::put($typeCacheKey, $type, Carbon::now()->addMonth()); + } + + return new self($filePath, $type); + } + public function index(string $query = null): LogIndex { if (! isset($this->_logIndexCache[$query])) { diff --git a/src/LogViewerService.php b/src/LogViewerService.php index af19b7d7..5bac689a 100755 --- a/src/LogViewerService.php +++ b/src/LogViewerService.php @@ -21,7 +21,7 @@ class LogViewerService protected mixed $hostsResolver; - protected function getFilePaths(): array + protected function getLaravelLogFilePaths(): array { // Because we'll use the base path as a parameter for `glob`, we should escape any // glob's special characters and treat those as actual characters of the path. @@ -70,6 +70,25 @@ protected function getFilePaths(): array return array_values(array_reverse($files)); } + protected function getHttpLogFilePaths(): array + { + $files = []; + + foreach (config('log-viewer.http_logs.include_files', []) as $pattern) { + $files = array_merge($files, $this->getFilePathsMatchingPattern($pattern)); + } + + foreach (config('log-viewer.http_logs.exclude_files', []) as $pattern) { + $files = array_diff($files, $this->getFilePathsMatchingPattern($pattern)); + } + + $files = array_map('realpath', $files); + + $files = array_filter($files, 'is_file'); + + return array_values(array_reverse($files)); + } + protected function getFilePathsMatchingPattern($pattern) { // The GLOB_BRACE flag is not available on some non GNU systems, like Solaris or Alpine Linux. @@ -88,10 +107,17 @@ public function basePathForLogs(): string public function getFiles(): LogFileCollection { if (! isset($this->_cachedFiles)) { - $this->_cachedFiles = (new LogFileCollection($this->getFilePaths())) + $laravelLogFiles = (new LogFileCollection($this->getLaravelLogFilePaths())) ->unique() ->map(fn ($filePath) => new LogFile($filePath)) ->values(); + + $httpLogFiles = (new LogFileCollection($this->getHttpLogFilePaths())) + ->unique() + ->map(fn ($filePath) => LogFile::makeAndGuessType($filePath)) + ->values(); + + $this->_cachedFiles = $laravelLogFiles->merge($httpLogFiles); } return $this->_cachedFiles; diff --git a/tests/Pest.php b/tests/Pest.php index c1f7df16..ec49261c 100644 --- a/tests/Pest.php +++ b/tests/Pest.php @@ -63,7 +63,8 @@ function dummyLogData(int $lines = null, string $type = LogFile::TYPE_LARAVEL): fn ($_) => match ($type) { LogFile::TYPE_LARAVEL => makeLaravelLogEntry(), LogFile::TYPE_HTTP_ACCESS => makeHttpAccessLogEntry(), - LogFile::TYPE_HTTP_ERROR => makeHttpErrorLogEntry(), + LogFile::TYPE_HTTP_ERROR_APACHE => makeHttpApacheErrorLogEntry(), + LogFile::TYPE_HTTP_ERROR_NGINX => makeHttpNginxErrorLogEntry(), }, range(1, $lines) )); @@ -94,7 +95,7 @@ function makeHttpAccessLogEntry(CarbonInterface $date = null, string $method = ' EOF; } -function makeHttpErrorLogEntry(CarbonInterface $date = null, string $module = null, string $level = null, int $pid = null, string $client = null, string $message = null): string +function makeHttpApacheErrorLogEntry(CarbonInterface $date = null, string $module = null, string $level = null, int $pid = null, string $client = null, string $message = null): string { $dateFormatted = $date instanceof CarbonInterface ? $date->format('D M d H:i:s.u Y') : now()->format('D M d H:i:s.u Y'); $module ??= 'php'; @@ -108,6 +109,22 @@ function makeHttpErrorLogEntry(CarbonInterface $date = null, string $module = nu EOF; } +function makeHttpNginxErrorLogEntry(CarbonInterface $date = null, string $level = null, string $message = null, string $client = null, string $server = null, string $request = null, string $host = null): string +{ + $dateFormatted = $date instanceof CarbonInterface ? $date->format('Y/m/d H:i:s') : now()->format('Y/m/d H:i:s'); + $level ??= 'error'; + $pid ??= rand(1, 9999); + $client ??= rand(1, 255).'.'.rand(1, 255).'.'.rand(1, 255).'.'.rand(1, 255); + $message ??= 'Testing log entry'; + $server ??= '127.0.0.1:80'; + $request ??= 'GET / HTTP/1.1'; + $host ??= 'localhost'; + + return <<text)->toBe($line) ->and($log->datetime->toDateTimeString())->toBe('2023-07-09 09:08:27') @@ -16,10 +16,24 @@ ->and($log->message)->toBe("script '/var/www/cgi-bin/cloud.php' not found or unable to stat"); }); +it('can parse an HTTP error log with client part missing', function () { + $line = "[Sun Jul 09 09:08:27.901758 2023] [php:error] [pid 116942] script '/var/www/cgi-bin/cloud.php' not found or unable to stat"; + + $log = new HttpApacheErrorLog($line); + + expect($log->text)->toBe($line) + ->and($log->datetime->toDateTimeString())->toBe('2023-07-09 09:08:27') + ->and($log->module)->toBe('php') + ->and($log->level)->toBe('error') + ->and($log->pid)->toBe(116942) + ->and($log->client)->toBe(null) + ->and($log->message)->toBe("script '/var/www/cgi-bin/cloud.php' not found or unable to stat"); +}); + it('can store the related file details', function () { $line = "[Sun Jul 09 09:08:27.901758 2023] [php:error] [pid 116942] [client 20.253.242.138:50173] script '/var/www/cgi-bin/cloud.php' not found or unable to stat"; - $log = new HttpErrorLog($line, $fileIdentifier = 'test-xyz.log', $filePosition = 123); + $log = new HttpApacheErrorLog($line, $fileIdentifier = 'test-xyz.log', $filePosition = 123); expect($log->fileIdentifier)->toBe($fileIdentifier) ->and($log->filePosition)->toBe($filePosition); @@ -28,7 +42,7 @@ it('handles missing details', function () { $line = ''; - $log = new HttpErrorLog($line); + $log = new HttpApacheErrorLog($line); expect($log->text)->toBe($line) ->and($log->datetime?->toDateTimeString())->toBe(null) @@ -42,7 +56,7 @@ it('strips empty chars at the end', function ($chars) { $line = "[Sun Jul 09 09:08:27.901758 2023] [php:error] [pid 116942] [client 20.253.242.138:50173] script '/var/www/cgi-bin/cloud.php' not found or unable to stat"; - $accessLog = new HttpErrorLog($line.$chars); + $accessLog = new HttpApacheErrorLog($line.$chars); expect($accessLog->text)->toBe($line); })->with([ diff --git a/tests/Unit/AccessLogs/HttpLogFilesTest.php b/tests/Unit/AccessLogs/HttpLogFilesTest.php new file mode 100644 index 00000000..4ed839e3 --- /dev/null +++ b/tests/Unit/AccessLogs/HttpLogFilesTest.php @@ -0,0 +1,34 @@ +set('log-viewer.http_logs.include_files', [__DIR__.'/Fixtures/*']); + + $files = LogViewer::getFiles(); + + expect($files)->toHaveCount(3) + + ->and($files[0])->toBeInstanceOf(LogFile::class) + ->and($files[0]->name)->toBe('errors_nginx_dummy.log') + ->and($files[0]->type)->toBe(LogFile::TYPE_HTTP_ERROR_NGINX) + ->and($files[0]->path)->toBe($error_nginx_dummy_path) + ->and($files[0]->size())->toBe(filesize($error_nginx_dummy_path)) + + ->and($files[1])->toBeInstanceOf(LogFile::class) + ->and($files[1]->name)->toBe('errors_dummy.log') + ->and($files[1]->type)->toBe(LogFile::TYPE_HTTP_ERROR_APACHE) + ->and($files[1]->path)->toBe($error_dummy_path) + ->and($files[1]->size())->toBe(filesize($error_dummy_path)) + + ->and($files[2])->toBeInstanceOf(LogFile::class) + ->and($files[2]->name)->toBe('access_dummy.log') + ->and($files[2]->type)->toBe(LogFile::TYPE_HTTP_ACCESS) + ->and($files[2]->path)->toBe($access_dummy_path) + ->and($files[2]->size())->toBe(filesize($access_dummy_path)); +}); diff --git a/tests/Unit/AccessLogs/HttpLogReaderTest.php b/tests/Unit/AccessLogs/HttpLogReaderTest.php index e2f480b8..3120f34a 100644 --- a/tests/Unit/AccessLogs/HttpLogReaderTest.php +++ b/tests/Unit/AccessLogs/HttpLogReaderTest.php @@ -1,7 +1,7 @@ toBeInstanceOf($expectedClass); })->with([ ['type' => LogFile::TYPE_HTTP_ACCESS, 'expectedClass' => HttpAccessLog::class], - ['type' => LogFile::TYPE_HTTP_ERROR, 'expectedClass' => HttpErrorLog::class], + ['type' => LogFile::TYPE_HTTP_ERROR_APACHE, 'expectedClass' => HttpApacheErrorLog::class], ]); it('can read one log at a time', function () { diff --git a/tests/Unit/AccessLogs/HttpNginxErrorLogTest.php b/tests/Unit/AccessLogs/HttpNginxErrorLogTest.php new file mode 100644 index 00000000..bb7aeb0c --- /dev/null +++ b/tests/Unit/AccessLogs/HttpNginxErrorLogTest.php @@ -0,0 +1,48 @@ +text)->toBe($line) + ->and($log->datetime->toDateTimeString())->toBe('2019-07-11 07:19:30') + ->and($log->level)->toBe('error') + ->and($log->message)->toBe('*18897816 open() "/local/nginx/static/ads.txt" failed (2: No such file or directory)') + ->and($log->client)->toBe('85.195.82.90') + ->and($log->server)->toBe('app.digitale-sammlungen.de') + ->and($log->request)->toBe('GET /ads.txt HTTP/1.1') + ->and($log->host)->toBe('app.digitale-sammlungen.de'); +}); + +it('can parse a less complex Nginx error log entry', function () { + $line = '2023/01/04 11:18:33 [alert] 95160#0: *1473 setsockopt(TCP_NODELAY) failed (22: Invalid argument) while keepalive, client: 127.0.0.1, server: 127.0.0.1:80'; + + $log = new HttpNginxErrorLog($line); + + expect($log->text)->toBe($line) + ->and($log->datetime->toDateTimeString())->toBe('2023-01-04 11:18:33') + ->and($log->level)->toBe('alert') + ->and($log->message)->toBe('*1473 setsockopt(TCP_NODELAY) failed (22: Invalid argument) while keepalive') + ->and($log->client)->toBe('127.0.0.1') + ->and($log->server)->toBe('127.0.0.1:80') + ->and($log->request)->toBe(null) + ->and($log->host)->toBe(null); +}); + +it('can parse a minimal log entry', function () { + $line = '2023/07/01 22:05:03 [warn] 21925#0: the "listen ... http2" directive is deprecated, use the "http2" directive instead in /Users/test/.config/valet/Nginx/blog9.test:9'; + + $log = new HttpNginxErrorLog($line); + + expect($log->text)->toBe($line) + ->and($log->datetime->toDateTimeString())->toBe('2023-07-01 22:05:03') + ->and($log->level)->toBe('warn') + ->and($log->message)->toBe('the "listen ... http2" directive is deprecated, use the "http2" directive instead in /Users/test/.config/valet/Nginx/blog9.test:9') + ->and($log->client)->toBe(null) + ->and($log->server)->toBe(null) + ->and($log->request)->toBe(null) + ->and($log->host)->toBe(null); +});