Working PHP 8 Multiprocessing (linux)

Author: Stefan

libs/ProcessCommunication/PHPCrawlerDocumentInfoQueue.php

@@ -83,7 +83,7 @@ public function addDocumentInfo(PHPCrawlerDocumentInfo $DocInfo): void
         $ser = serialize($DocInfo);
 
         $this->PDO->exec('BEGIN EXCLUSIVE TRANSACTION');
-        $this->preparedInsertStatement->bindParam(1, $ser, PDO::PARAM_LOB);
+        $this->preparedInsertStatement->bindParam(1, $ser, PDO::PARAM_STR);
         $this->preparedInsertStatement->execute();
         $this->preparedSelectStatement->closeCursor();
         $this->PDO->exec('COMMIT');
@@ -97,7 +97,7 @@ public function getNextDocumentInfo()
         $this->createPreparedStatements();
 
         $this->preparedSelectStatement->execute();
-        $this->preparedSelectStatement->bindColumn('document_info', $doc_info, PDO::PARAM_LOB);
+        $this->preparedSelectStatement->bindColumn('document_info', $doc_info, PDO::PARAM_STR);
         $this->preparedSelectStatement->bindColumn('id', $id);
         $row = $this->preparedSelectStatement->fetch(PDO::FETCH_BOUND);
         $this->preparedSelectStatement->closeCursor();

libs/UrlCache/PHPCrawlerURLCacheBase.php

@@ -131,7 +131,10 @@ public function addLinkPriority($regex, $level): void
         $this->url_priorities[$c]['level'] = trim($level);
 
         // Sort url-priortie-array so that high priority-levels come firts.
-        PHPCrawlerUtils::sort2dArray($this->url_priorities, 'level', SORT_DESC);
+        #PHPCrawlerUtils::sort2dArray($this->url_priorities, 'level', SORT_DESC);
+        $key_values = array_column($this->url_priorities, 'level');
+        array_multisort($key_values, SORT_ASC, $this->url_priorities);
+
     }
 
     /**

libs/Utils/PHPCrawlerUtils.php

@@ -521,13 +521,15 @@ public static function sort2dArray(&$array, $sort_args): void
     {
         $args = func_get_args();
 
-        // F?r jedes zu sortierende Feld ein eigenes Array bilden
+
+        // Form a separate array for each field to be sorted
         foreach ($array as $fieldKey => $field) {
+
             for ($x = 1, $xMax = count($args); $x < $xMax; $x++) {
-                // Ist das Argument ein String, sprich ein Sortier-Feld?
+                // Is the argument a string, i.e. a sort field?
+
                 if (is_string($args[$x])) {
                     $value = $array[$field][$args[$x]];
-
                     ${$args[$x]}[] = $value;
                 }
             }

multiprocessing_example.php

@@ -61,6 +61,7 @@ public function handleDocumentInfo($DocInfo): int
         echo $lb;
 
         flush();
+        return 0;
     }
 }
 
@@ -85,7 +86,8 @@ public function handleDocumentInfo($DocInfo): int
 // "http://php.net/manual/en/" followed by  "mysql" somewhere.
 // So we add a corresponding follow-rule to the crawler.
 $crawler->addURLFollowRule('#^http://php.net/manual/en/.*mysql[^a-z]# i');
-
+$crawler->setRequestLimit(10); // Set the page-limit to 50 for testing
+$crawler->addLinkPriority('/php/i', 1);
 // That's it, start crawling using 5 processes
 $crawler->goMultiProcessed(5);