Skip to content

Commit

Permalink
MetadataCollection::import(): report errors in ERRMODE_PASS mode
Browse files Browse the repository at this point in the history
  • Loading branch information
@zozlak
zozlak committed Jan 9, 2022
1 parent 5dea903 commit 26fdf4c
Show file tree
Hide file tree
Showing 4 changed files with 39 additions and 30 deletions.
2 changes: 1 addition & 1 deletion src/acdhOeaw/arche/lib/ingest/Indexer.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -524,7 +524,7 @@ public function import(string $errorMode = self::ERRMODE_FAIL,
}
}
if ($errorsCount > 0 && $errorMode === self::ERRMODE_PASS) {
throw new IndexerException('There was at least one error during the import', 1, null, $allRepoRes);
throw new IndexerException('There was at least one error during the import', IndexerException::ERROR_DURING_IMPORT, null, $allRepoRes);
}
return $allRepoRes;
}
Expand Down
2 changes: 2 additions & 0 deletions src/acdhOeaw/arche/lib/ingest/IndexerException.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,8 @@
*/
class IndexerException extends Exception {

const ERROR_DURING_IMPORT = 1;

/**
* A collection of already processed resources
* @var array<RepoResource>
Expand Down
23 changes: 15 additions & 8 deletions src/acdhOeaw/arche/lib/ingest/MetadataCollection.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -275,24 +275,31 @@ function (RepoResource $repoRes) use ($res, $n, $N) {
return $promise1;
};

$allRepoRes = [];
$errorsCount = 0;
$chunkSize = $this->autoCommit > 0 ? $this->autoCommit : count($toBeImported);
$allRepoRes = [];
$errors = '';
$chunkSize = $this->autoCommit > 0 ? $this->autoCommit : count($toBeImported);
for ($i = 0; $i < count($toBeImported); $i += $chunkSize) {
if ($i > 0 && $errorsCount === 0) {
if ($i > 0 && empty($errors)) {
echo self::$debug ? "Autocommit\n" : '';
$this->repo->commit();
$this->repo->begin();
}
$chunk = array_slice($toBeImported, $i, $chunkSize);
$chunkRepoRes = $this->repo->map($chunk, $f, $concurrency, $mapErrorMode);
foreach ($chunkRepoRes as $j) {
$errorsCount += (int) ($j instanceof Exception);
foreach ($chunkRepoRes as $n => $j) {
if ($j instanceof Exception) {
$msg = $j instanceof ClientException ? $j->getResponse()->getBody() : $j->getMessage();
$msg = $chunk[$n]->getUri() . ": " . $msg;
$errors .= "\t$msg\n";
if (self::$debug) {
echo "\tERROR while processing $msg\n";
}
}
}
$allRepoRes = array_merge($allRepoRes, $chunkRepoRes);
}
if ($errorsCount > 0 && $errorMode === self::ERRMODE_PASS) {
throw new IndexerException('There was at least one error during the import');
if (!empty($errors) && $errorMode === self::ERRMODE_PASS) {
throw new IndexerException("There was at least one error during the import:\n.$errors", IndexerException::ERROR_DURING_IMPORT);
}

return $allRepoRes;
Expand Down
42 changes: 21 additions & 21 deletions tests/MetadataCollectionTest.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ static public function setUpBeforeClass(): void {
*/
public function testSimple(): void {
self::$test = 'testSimple';

$graph = new MetadataCollection(self::$repo, __DIR__ . '/data/graph-small.ttl');
self::$repo->begin();
$indRes = $graph->import('https://id.acdh.oeaw.ac.at/', MetadataCollection::SKIP);
Expand All @@ -65,7 +65,7 @@ public function testSimple(): void {
*/
public function testSimpleDouble(): void {
self::$test = 'testSimpleDouble';

$graph = new MetadataCollection(self::$repo, __DIR__ . '/data/graph-small.ttl');
self::$repo->begin();
$indRes1 = $graph->import('https://id.acdh.oeaw.ac.at/', MetadataCollection::SKIP);
Expand Down Expand Up @@ -97,7 +97,7 @@ public function testSimpleDouble(): void {
*/
public function testLarge(): void {
self::$test = 'testLarge';

$graph = new MetadataCollection(self::$repo, __DIR__ . '/data/graph-large.ttl');
self::$repo->begin();
$indRes = $graph->import('https://id.acdh.oeaw.ac.at/', MetadataCollection::SKIP);
Expand All @@ -112,7 +112,7 @@ public function testLarge(): void {
*/
public function testCycle(): void {
self::$test = 'testCycle';

$graph = new MetadataCollection(self::$repo, __DIR__ . '/data/graph-cycle.ttl');
self::$repo->begin();
$indRes = $graph->import('http://some.id', MetadataCollection::SKIP);
Expand All @@ -127,7 +127,7 @@ public function testCycle(): void {
*/
public function testBNodes(): void {
self::$test = 'testBNodes';

$graph = new MetadataCollection(self::$repo, __DIR__ . '/data/bnodes.ttl');
self::$repo->begin();
$indRes = $graph->import('https://id.acdh.oeaw.ac.at/', MetadataCollection::SKIP);
Expand All @@ -142,7 +142,7 @@ public function testBNodes(): void {
*/
public function testAutoRefsCreation(): void {
self::$test = 'testAutoRefsCreation';

$graph = new MetadataCollection(self::$repo, __DIR__ . '/data/graph-autorefs.ttl');
self::$repo->begin();
$indRes = $graph->import('https://id.acdh.oeaw.ac.at/', MetadataCollection::SKIP);
Expand All @@ -157,7 +157,7 @@ public function testAutoRefsCreation(): void {
*/
public function testBasicResources(): void {
self::$test = 'testBasicResources';

$graph = new MetadataCollection(self::$repo, __DIR__ . '/data/basicResources.ttl');
self::$repo->begin();
$indRes = $graph->import('https://id.acdh.oeaw.ac.at/', MetadataCollection::SKIP);
Expand Down Expand Up @@ -186,7 +186,7 @@ public function testBasicResources(): void {
*/
public function testBig(): void {
self::$test = 'testBig';

$graph = new MetadataCollection(self::$repo, __DIR__ . '/data/schnitzler-diaries.rdf');
self::$repo->begin();
$indRes = $graph->import('https://id.acdh.oeaw.ac.at/', MetadataCollection::SKIP);
Expand All @@ -201,7 +201,7 @@ public function testBig(): void {
*/
public function testImportSingleOutNmsp(): void {
self::$test = 'testImportSingleOutNmsp';

$graph = new MetadataCollection(self::$repo, __DIR__ . '/data/basicResources.ttl');
self::$repo->begin();
$this->expectException(\InvalidArgumentException::class);
Expand All @@ -216,7 +216,7 @@ public function testImportSingleOutNmsp(): void {
*/
public function testImportWrongErrorMode(): void {
self::$test = 'testImportWrongErrorMode';

$graph = new MetadataCollection(self::$repo, __DIR__ . '/data/basicResources.ttl');
self::$repo->begin();
$this->expectException(\InvalidArgumentException::class);
Expand All @@ -225,19 +225,19 @@ public function testImportWrongErrorMode(): void {
$this->noteResources($indRes);
self::$repo->commit();
}

/**
* @group metadataCollection
*/
public function testImportErrorMode(): void {
self::$test = 'testImportErrorMode';

// ERRMODE_INCLUDE
$graph = new MetadataCollection(self::$repo, __DIR__ . '/data/errmode.ttl');
self::$repo->begin();
$indRes = $graph->import('https://id.acdh.oeaw.ac.at/', MetadataCollection::SKIP, MetadataCollection::ERRMODE_INCLUDE);
self::$repo->rollback();

$classes = [];
foreach ($indRes as $i) {
$classes[] = get_class($i);
Expand All @@ -246,15 +246,15 @@ public function testImportErrorMode(): void {
$this->assertCount(2, $classes);
$this->assertContains(RepoResource::class, $classes);
$this->assertContains(ClientException::class, $classes);

// ERRMODE_PASS
$graph = new MetadataCollection(self::$repo, __DIR__ . '/data/errmode.ttl');
$graph = new MetadataCollection(self::$repo, __DIR__ . '/data/errmode.ttl');
self::$repo->begin();
try {
$indRes = $graph->import('https://id.acdh.oeaw.ac.at/', MetadataCollection::SKIP, MetadataCollection::ERRMODE_PASS);
$this->assertTrue(false);
} catch(IndexerException $e) {
$this->assertEquals('There was at least one error during the import', $e->getMessage());
} catch (IndexerException $e) {
$this->assertStringStartsWith('There was at least one error during the import', $e->getMessage());
}
$this->assertInstanceOf(RepoResource::class, self::$repo->getResourceById('https://id.acdh.oeaw.ac.at/id1'));
try {
Expand All @@ -264,14 +264,14 @@ public function testImportErrorMode(): void {
$this->assertTrue(true);
}
self::$repo->rollback();

// ERRMODE_FAIL
$graph = new MetadataCollection(self::$repo, __DIR__ . '/data/errmode.ttl');
$graph = new MetadataCollection(self::$repo, __DIR__ . '/data/errmode.ttl');
self::$repo->begin();
try {
$indRes = $graph->import('https://id.acdh.oeaw.ac.at/', MetadataCollection::SKIP, MetadataCollection::ERRMODE_FAIL);
$this->assertTrue(false);
} catch(ClientException $e) {
} catch (ClientException $e) {
$this->assertStringContainsString('Wrong property value', $e->getMessage());
}
// can't test for resource one as ingestion order is unknown
Expand All @@ -282,5 +282,5 @@ public function testImportErrorMode(): void {
$this->assertTrue(true);
}
self::$repo->rollback();
}
}
}

0 comments on commit 26fdf4c

Please sign in to comment.