Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions lib/composer/composer/autoload_classmap.php
Original file line number Diff line number Diff line change
Expand Up @@ -282,6 +282,7 @@
'OCP\\DB\\IPreparedStatement' => $baseDir . '/lib/public/DB/IPreparedStatement.php',
'OCP\\DB\\IResult' => $baseDir . '/lib/public/DB/IResult.php',
'OCP\\DB\\ISchemaWrapper' => $baseDir . '/lib/public/DB/ISchemaWrapper.php',
'OCP\\DB\\QueryBuilder\\ConflictResolutionMode' => $baseDir . '/lib/public/DB/QueryBuilder/ConflictResolutionMode.php',
'OCP\\DB\\QueryBuilder\\ICompositeExpression' => $baseDir . '/lib/public/DB/QueryBuilder/ICompositeExpression.php',
'OCP\\DB\\QueryBuilder\\IExpressionBuilder' => $baseDir . '/lib/public/DB/QueryBuilder/IExpressionBuilder.php',
'OCP\\DB\\QueryBuilder\\IFunctionBuilder' => $baseDir . '/lib/public/DB/QueryBuilder/IFunctionBuilder.php',
Expand Down
1 change: 1 addition & 0 deletions lib/composer/composer/autoload_static.php
Original file line number Diff line number Diff line change
Expand Up @@ -331,6 +331,7 @@ class ComposerStaticInit749170dad3f5e7f9ca158f5a9f04f6a2
'OCP\\DB\\IPreparedStatement' => __DIR__ . '/../../..' . '/lib/public/DB/IPreparedStatement.php',
'OCP\\DB\\IResult' => __DIR__ . '/../../..' . '/lib/public/DB/IResult.php',
'OCP\\DB\\ISchemaWrapper' => __DIR__ . '/../../..' . '/lib/public/DB/ISchemaWrapper.php',
'OCP\\DB\\QueryBuilder\\ConflictResolutionMode' => __DIR__ . '/../../..' . '/lib/public/DB/QueryBuilder/ConflictResolutionMode.php',
'OCP\\DB\\QueryBuilder\\ICompositeExpression' => __DIR__ . '/../../..' . '/lib/public/DB/QueryBuilder/ICompositeExpression.php',
'OCP\\DB\\QueryBuilder\\IExpressionBuilder' => __DIR__ . '/../../..' . '/lib/public/DB/QueryBuilder/IExpressionBuilder.php',
'OCP\\DB\\QueryBuilder\\IFunctionBuilder' => __DIR__ . '/../../..' . '/lib/public/DB/QueryBuilder/IFunctionBuilder.php',
Expand Down
6 changes: 6 additions & 0 deletions lib/private/DB/QueryBuilder/ExtendedQueryBuilder.php
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@

use OC\DB\Exceptions\DbalException;
use OCP\DB\IResult;
use OCP\DB\QueryBuilder\ConflictResolutionMode;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\IDBConnection;

Expand Down Expand Up @@ -306,4 +307,9 @@ public function getOutputColumns(): array {
public function prefixTableName(string $table): string {
return $this->builder->prefixTableName($table);
}

public function forUpdate(ConflictResolutionMode $conflictResolutionMode = ConflictResolutionMode::Ordinary): self {
$this->builder->forUpdate($conflictResolutionMode);
return $this;
}
}
10 changes: 10 additions & 0 deletions lib/private/DB/QueryBuilder/QueryBuilder.php
Original file line number Diff line number Diff line change
Expand Up @@ -20,12 +20,14 @@
use OC\DB\QueryBuilder\FunctionBuilder\SqliteFunctionBuilder;
use OC\SystemConfig;
use OCP\DB\IResult;
use OCP\DB\QueryBuilder\ConflictResolutionMode;
use OCP\DB\QueryBuilder\ICompositeExpression;
use OCP\DB\QueryBuilder\ILiteral;
use OCP\DB\QueryBuilder\IParameter;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\DB\QueryBuilder\IQueryFunction;
use OCP\IDBConnection;
use Override;
use Psr\Log\LoggerInterface;

class QueryBuilder implements IQueryBuilder {
Expand Down Expand Up @@ -1400,4 +1402,12 @@ public function runAcrossAllShards(): self {
return $this;
}

#[Override]
public function forUpdate(ConflictResolutionMode $conflictResolutionMode = ConflictResolutionMode::Ordinary): self {
match ($conflictResolutionMode) {
ConflictResolutionMode::Ordinary => $this->queryBuilder->forUpdate(),
ConflictResolutionMode::SkipLocked => $this->queryBuilder->forUpdate(\Doctrine\DBAL\Query\ForUpdate\ConflictResolutionMode::SKIP_LOCKED),
};
return $this;
}
}
147 changes: 111 additions & 36 deletions lib/private/Files/Cache/Propagator.php
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
use OC\DB\Exceptions\DbalException;
use OC\Files\Storage\LocalRootStorage;
use OC\Files\Storage\Wrapper\Encryption;
use OCP\DB\QueryBuilder\ILiteral;
use OCP\DB\QueryBuilder\IQueryBuilder;
use OCP\Files\Cache\IPropagator;
use OCP\Files\Storage\IReliableEtagStorage;
Expand Down Expand Up @@ -87,12 +88,11 @@
}

$parentHashes = array_map('md5', $parents);
sort($parentHashes); // Ensure rows are always locked in the same order
$etag = uniqid(); // since we give all folders the same etag we don't ask the storage for the etag

$builder = $this->connection->getQueryBuilder();
$hashParams = array_map(function ($hash) use ($builder) {
return $builder->expr()->literal($hash);
}, $parentHashes);
$hashParams = array_map(static fn (string $hash): ILiteral => $builder->expr()->literal($hash), $parentHashes);

$builder->update('filecache')
->set('mtime', $builder->func()->greatest('mtime', $builder->createNamedParameter($time, IQueryBuilder::PARAM_INT)))
Expand Down Expand Up @@ -133,9 +133,27 @@

for ($i = 0; $i < self::MAX_RETRIES; $i++) {
try {
$builder->executeStatement();
if ($this->connection->getDatabaseProvider() !== IDBConnection::PLATFORM_SQLITE) {
$this->connection->beginTransaction();
// Lock all the rows first with a SELECT FOR UPDATE ordered by path_hash
$forUpdate = $this->connection->getQueryBuilder();
$forUpdate->select('fileid')
->from('filecache')
->where($forUpdate->expr()->eq('storage', $forUpdate->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($forUpdate->expr()->in('path_hash', $hashParams))
->orderBy('path_hash')
->forUpdate()
->executeQuery();
$builder->executeStatement();
$this->connection->commit();
} else {
$builder->executeStatement();
}
break;
} catch (DbalException $e) {
if ($this->connection->getDatabaseProvider() !== IDBConnection::PLATFORM_SQLITE) {
$this->connection->rollBack();
}
if (!$e->isRetryable()) {
throw $e;
}
Expand Down Expand Up @@ -194,40 +212,97 @@
}
$this->inBatch = false;

$this->connection->beginTransaction();

$query = $this->connection->getQueryBuilder();
$storageId = (int)$this->storage->getStorageCache()->getNumericId();

$query->update('filecache')
->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time')))
->set('etag', $query->expr()->literal(uniqid()))
->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash')));

$sizeQuery = $this->connection->getQueryBuilder();
$sizeQuery->update('filecache')
->set('size', $sizeQuery->func()->add('size', $sizeQuery->createParameter('size')))
->where($query->expr()->eq('storage', $sizeQuery->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->eq('path_hash', $sizeQuery->createParameter('hash')))
->andWhere($sizeQuery->expr()->gt('size', $sizeQuery->createNamedParameter(-1, IQueryBuilder::PARAM_INT)));

foreach ($this->batch as $item) {
$query->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT);
$query->setParameter('hash', $item['hash']);

$query->executeStatement();

if ($item['size']) {
$sizeQuery->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT);
$sizeQuery->setParameter('hash', $item['hash']);

$sizeQuery->executeStatement();
// Ensure rows are always locked in the same order
uasort($this->batch, static fn (array $a, array $b) => $a['hash'] <=> $b['hash']);

try {
$this->connection->beginTransaction();

$query = $this->connection->getQueryBuilder();
$storageId = (int)$this->storage->getStorageCache()->getNumericId();

if ($this->connection->getDatabaseProvider() !== IDBConnection::PLATFORM_SQLITE) {
// Lock the rows before updating then with a SELECT FOR UPDATE
// The select also allow us to fetch the fileid and then use these in the UPDATE
// queries as a faster lookup than the path_hash
$hashes = array_map(static fn (array $a): string => $a['hash'], $this->batch);

foreach (array_chunk($hashes, 1000) as $hashesChunk) {
$query = $this->connection->getQueryBuilder();
$result = $query->select('fileid', 'path', 'path_hash', 'size')
->from('filecache')
->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->in('path_hash', $query->createNamedParameter($hashesChunk, IQueryBuilder::PARAM_STR_ARRAY)))
->orderBy('path_hash')
->forUpdate()
->executeQuery();

$query = $this->connection->getQueryBuilder();
$query->update('filecache')
->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time')))
->set('etag', $query->expr()->literal(uniqid()))
->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->eq('fileid', $query->createParameter('fileid')));

$queryWithSize = $this->connection->getQueryBuilder();
$queryWithSize->update('filecache')
->set('mtime', $queryWithSize->func()->greatest('mtime', $queryWithSize->createParameter('time')))
->set('etag', $queryWithSize->expr()->literal(uniqid()))
->set('size', $queryWithSize->func()->add('size', $queryWithSize->createParameter('size')))
->where($queryWithSize->expr()->eq('storage', $queryWithSize->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($queryWithSize->expr()->eq('fileid', $queryWithSize->createParameter('fileid')));

while ($row = $result->fetchAssociative()) {

Check failure on line 255 in lib/private/Files/Cache/Propagator.php

View workflow job for this annotation

GitHub Actions / static-code-analysis

UndefinedInterfaceMethod

lib/private/Files/Cache/Propagator.php:255:29: UndefinedInterfaceMethod: Method OCP\DB\IResult::fetchAssociative does not exist (see https://psalm.dev/181)
$item = $this->batch[$row['path']];
if ($item['size'] && $row['size'] > -1) {
$queryWithSize->setParameter('fileid', $row['fileid'], IQueryBuilder::PARAM_INT)
->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT)
->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT)
->executeStatement();
} else {
$query->setParameter('fileid', $row['fileid'], IQueryBuilder::PARAM_INT)
->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT)
->executeStatement();
}
}
}
} else {
// No FOR UPDATE support in Sqlite, but instead the whole table is locked
$query = $this->connection->getQueryBuilder();
$query->update('filecache')
->set('mtime', $query->func()->greatest('mtime', $query->createParameter('time')))
->set('etag', $query->expr()->literal(uniqid()))
->where($query->expr()->eq('storage', $query->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($query->expr()->eq('path_hash', $query->createParameter('hash')));

$queryWithSize = $this->connection->getQueryBuilder();
$queryWithSize->update('filecache')
->set('mtime', $queryWithSize->func()->greatest('mtime', $queryWithSize->createParameter('time')))
->set('etag', $queryWithSize->expr()->literal(uniqid()))
->set('size', $queryWithSize->func()->add('size', $queryWithSize->createParameter('size')))
->where($queryWithSize->expr()->eq('storage', $queryWithSize->createNamedParameter($storageId, IQueryBuilder::PARAM_INT)))
->andWhere($queryWithSize->expr()->eq('path_hash', $queryWithSize->createParameter('hash')));

foreach ($this->batch as $item) {
if ($item['size']) {
$queryWithSize->setParameter('hash', $item['hash'], IQueryBuilder::PARAM_STR)
->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT)
->setParameter('size', $item['size'], IQueryBuilder::PARAM_INT)
->executeStatement();
} else {
$query->setParameter('hash', $item['hash'], IQueryBuilder::PARAM_STR)
->setParameter('time', $item['time'], IQueryBuilder::PARAM_INT)
->executeStatement();
}
}
}
}

$this->batch = [];
$this->batch = [];

$this->connection->commit();
$this->connection->commit();
} catch (\Exception $e) {
$this->connection->rollback();
throw $e;
}
}
}
26 changes: 26 additions & 0 deletions lib/public/DB/QueryBuilder/ConflictResolutionMode.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<?php

declare(strict_types=1);

/**
* SPDX-FileCopyrightText: 2026 Nextcloud GmbH and Nextcloud contributors
* SPDX-License-Identifier: AGPL-3.0-or-later
*/

namespace OCP\DB\QueryBuilder;

/**
* Conflict resolution mode for "FOR UPDATE" select queries.
*
* @since 34.0.0
*/
enum ConflictResolutionMode {
/**
* Wait for the row to be unlocked.
*/
case Ordinary;
/**
* Skip the row if it is locked.
*/
case SkipLocked;
}
8 changes: 8 additions & 0 deletions lib/public/DB/QueryBuilder/IQueryBuilder.php
Original file line number Diff line number Diff line change
Expand Up @@ -1108,4 +1108,12 @@ public function runAcrossAllShards(): self;
* @since 30.0.0
*/
public function getOutputColumns(): array;

/**
* Locks the queried rows for a subsequent update.
*
* @return $this
* @since 34.0.0
*/
public function forUpdate(ConflictResolutionMode $conflictResolutionMode = ConflictResolutionMode::Ordinary): self;
}
3 changes: 1 addition & 2 deletions tests/lib/Files/Cache/PropagatorTest.php
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,7 @@
* @group DB
*/
class PropagatorTest extends TestCase {
/** @var IStorage */
private $storage;
private IStorage $storage;

protected function setUp(): void {
parent::setUp();
Expand Down
Loading