diff --git a/UPGRADE.md b/UPGRADE.md index 90e306eadb4..8dcf4108a4a 100644 --- a/UPGRADE.md +++ b/UPGRADE.md @@ -666,6 +666,15 @@ Use `toIterable()` instead. # Upgrade to 2.17 +## Deprecate annotations classes for named queries + +The following classes have been deprecated: + +* `Doctrine\ORM\Mapping\NamedNativeQueries` +* `Doctrine\ORM\Mapping\NamedNativeQuery` +* `Doctrine\ORM\Mapping\NamedQueries` +* `Doctrine\ORM\Mapping\NamedQuery` + ## Deprecate `Doctrine\ORM\Query\Exec\AbstractSqlExecutor::_sqlStatements` Use `Doctrine\ORM\Query\Exec\AbstractSqlExecutor::sqlStatements` instead. diff --git a/docs/en/reference/working-with-objects.rst b/docs/en/reference/working-with-objects.rst index 640cdb60f4d..cc107e6e174 100644 --- a/docs/en/reference/working-with-objects.rst +++ b/docs/en/reference/working-with-objects.rst @@ -711,6 +711,23 @@ and these associations are mapped as EAGER, they will automatically be loaded together with the entity being queried and is thus immediately available to your application. +Eager Loading can also be configured at runtime through +``AbstractQuery::setFetchMode`` in DQL or Native Queries. + +Eager loading for many-to-one and one-to-one associations is using either a +LEFT JOIN or a second query for fetching the related entity eagerly. + +Eager loading for many-to-one associations uses a second query to load +the collections for several entities at the same time. + +When many-to-many, one-to-one or one-to-many associations are eagerly loaded, +then the global batch size configuration is used to avoid IN(?) queries with +too many arguments. The default batch size is 100 and can be changed with +``Configuration::setEagerFetchBatchSize()``. + +For eagerly loaded Many-To-Many associations one query has to be made for each +collection. + By Lazy Loading ~~~~~~~~~~~~~~~ diff --git a/lib/Doctrine/ORM/Configuration.php b/lib/Doctrine/ORM/Configuration.php index 63c39d42de2..b30764eb2b9 100644 --- a/lib/Doctrine/ORM/Configuration.php +++ b/lib/Doctrine/ORM/Configuration.php @@ -636,4 +636,14 @@ public function isRejectIdCollisionInIdentityMapEnabled(): bool { return true; } + + public function setEagerFetchBatchSize(int $batchSize = 100): void + { + $this->attributes['fetchModeSubselectBatchSize'] = $batchSize; + } + + public function getEagerFetchBatchSize(): int + { + return $this->attributes['fetchModeSubselectBatchSize'] ?? 100; + } } diff --git a/lib/Doctrine/ORM/Persisters/Entity/BasicEntityPersister.php b/lib/Doctrine/ORM/Persisters/Entity/BasicEntityPersister.php index a338805b637..5a2fc92a315 100644 --- a/lib/Doctrine/ORM/Persisters/Entity/BasicEntityPersister.php +++ b/lib/Doctrine/ORM/Persisters/Entity/BasicEntityPersister.php @@ -1220,7 +1220,7 @@ protected function getSelectColumnsSQL(): string } $isAssocToOneInverseSide = $assoc->isToOne() && ! $assoc->isOwningSide(); - $isAssocFromOneEager = ! $assoc->isManyToMany() && $assoc->fetch === ClassMetadata::FETCH_EAGER; + $isAssocFromOneEager = $assoc->isToOne() && $assoc->fetch === ClassMetadata::FETCH_EAGER; if (! ($isAssocFromOneEager || $isAssocToOneInverseSide)) { continue; diff --git a/lib/Doctrine/ORM/Query/QueryException.php b/lib/Doctrine/ORM/Query/QueryException.php index 1f0e4728936..ae945b167fe 100644 --- a/lib/Doctrine/ORM/Query/QueryException.php +++ b/lib/Doctrine/ORM/Query/QueryException.php @@ -117,6 +117,14 @@ public static function iterateWithFetchJoinNotAllowed(AssociationMapping $assoc) ); } + public static function eagerFetchJoinWithNotAllowed(string $sourceEntity, string $fieldName): self + { + return new self( + 'Associations with fetch-mode=EAGER may not be using WITH conditions in + "' . $sourceEntity . '#' . $fieldName . '".', + ); + } + public static function iterateWithMixedResultNotAllowed(): self { return new self('Iterating a query with mixed results (using scalars) is not supported.'); diff --git a/lib/Doctrine/ORM/Query/SqlWalker.php b/lib/Doctrine/ORM/Query/SqlWalker.php index eab73969b4b..f6f94347e36 100644 --- a/lib/Doctrine/ORM/Query/SqlWalker.php +++ b/lib/Doctrine/ORM/Query/SqlWalker.php @@ -894,7 +894,9 @@ public function walkJoinAssociationDeclaration( } } - $targetTableJoin = null; + if ($relation->fetch === ClassMetadata::FETCH_EAGER && $condExpr !== null) { + throw QueryException::eagerFetchJoinWithNotAllowed($assoc->sourceEntity, $assoc->fieldName); + } // This condition is not checking ClassMetadata::MANY_TO_ONE, because by definition it cannot // be the owning side and previously we ensured that $assoc is always the owning side of the associations. diff --git a/lib/Doctrine/ORM/UnitOfWork.php b/lib/Doctrine/ORM/UnitOfWork.php index d626b7210ca..4cfcb346224 100644 --- a/lib/Doctrine/ORM/UnitOfWork.php +++ b/lib/Doctrine/ORM/UnitOfWork.php @@ -33,6 +33,7 @@ use Doctrine\ORM\Mapping\AssociationMapping; use Doctrine\ORM\Mapping\ClassMetadata; use Doctrine\ORM\Mapping\MappingException; +use Doctrine\ORM\Mapping\ToManyInverseSideMapping; use Doctrine\ORM\Persisters\Collection\CollectionPersister; use Doctrine\ORM\Persisters\Collection\ManyToManyPersister; use Doctrine\ORM\Persisters\Collection\OneToManyPersister; @@ -50,6 +51,7 @@ use Throwable; use UnexpectedValueException; +use function array_chunk; use function array_combine; use function array_diff_key; use function array_filter; @@ -292,6 +294,9 @@ class UnitOfWork implements PropertyChangedListener */ private array $eagerLoadingEntities = []; + /** @var array> */ + private array $eagerLoadingCollections = []; + protected bool $hasCache = false; /** @@ -2218,6 +2223,7 @@ public function clear(): void $this->pendingCollectionElementRemovals = $this->visitedCollections = $this->eagerLoadingEntities = + $this->eagerLoadingCollections = $this->orphanRemovals = []; if ($this->evm->hasListeners(Events::onClear)) { @@ -2352,6 +2358,10 @@ public function createEntity(string $className, array $data, array &$hints = []) continue; } + if (! isset($hints['fetchMode'][$class->name][$field])) { + $hints['fetchMode'][$class->name][$field] = $assoc->fetch; + } + $targetClass = $this->em->getClassMetadata($assoc->targetEntity); switch (true) { @@ -2416,10 +2426,6 @@ public function createEntity(string $className, array $data, array &$hints = []) break; } - if (! isset($hints['fetchMode'][$class->name][$field])) { - $hints['fetchMode'][$class->name][$field] = $assoc->fetch; - } - // Foreign key is set // Check identity map first // FIXME: Can break easily with composite keys if join column values are in @@ -2514,9 +2520,13 @@ public function createEntity(string $className, array $data, array &$hints = []) $reflField = $class->reflFields[$field]; $reflField->setValue($entity, $pColl); - if ($assoc->fetch === ClassMetadata::FETCH_EAGER) { - $this->loadCollection($pColl); - $pColl->takeSnapshot(); + if ($hints['fetchMode'][$class->name][$field] === ClassMetadata::FETCH_EAGER) { + if ($assoc->isOneToMany()) { + $this->scheduleCollectionForBatchLoading($pColl, $class); + } elseif ($assoc->isManyToMany()) { + $this->loadCollection($pColl); + $pColl->takeSnapshot(); + } } $this->originalEntityData[$oid][$field] = $pColl; @@ -2532,7 +2542,7 @@ public function createEntity(string $className, array $data, array &$hints = []) public function triggerEagerLoads(): void { - if (! $this->eagerLoadingEntities) { + if (! $this->eagerLoadingEntities && ! $this->eagerLoadingCollections) { return; } @@ -2545,11 +2555,69 @@ public function triggerEagerLoads(): void continue; } - $class = $this->em->getClassMetadata($entityName); + $class = $this->em->getClassMetadata($entityName); + $batches = array_chunk($ids, $this->em->getConfiguration()->getEagerFetchBatchSize()); - $this->getEntityPersister($entityName)->loadAll( - array_combine($class->identifier, [array_values($ids)]), - ); + foreach ($batches as $batchedIds) { + $this->getEntityPersister($entityName)->loadAll( + array_combine($class->identifier, [$batchedIds]), + ); + } + } + + $eagerLoadingCollections = $this->eagerLoadingCollections; // avoid recursion + $this->eagerLoadingCollections = []; + + foreach ($eagerLoadingCollections as $group) { + $this->eagerLoadCollections($group['items'], $group['mapping']); + } + } + + /** + * Load all data into the given collections, according to the specified mapping + * + * @param PersistentCollection[] $collections + */ + private function eagerLoadCollections(array $collections, ToManyInverseSideMapping $mapping): void + { + $targetEntity = $mapping->targetEntity; + $class = $this->em->getClassMetadata($mapping->sourceEntity); + $mappedBy = $mapping->mappedBy; + + $batches = array_chunk($collections, $this->em->getConfiguration()->getEagerFetchBatchSize(), true); + + foreach ($batches as $collectionBatch) { + $entities = []; + + foreach ($collectionBatch as $collection) { + $entities[] = $collection->getOwner(); + } + + $found = $this->getEntityPersister($targetEntity)->loadAll([$mappedBy => $entities]); + + $targetClass = $this->em->getClassMetadata($targetEntity); + $targetProperty = $targetClass->getReflectionProperty($mappedBy); + assert($targetProperty !== null); + + foreach ($found as $targetValue) { + $sourceEntity = $targetProperty->getValue($targetValue); + + $id = $this->identifierFlattener->flattenIdentifier($class, $class->getIdentifierValues($sourceEntity)); + $idHash = implode(' ', $id); + + if ($mapping->indexBy !== null) { + $indexByProperty = $targetClass->getReflectionProperty($mapping->indexBy); + assert($indexByProperty !== null); + $collectionBatch[$idHash]->hydrateSet($indexByProperty->getValue($targetValue), $targetValue); + } else { + $collectionBatch[$idHash]->add($targetValue); + } + } + } + + foreach ($collections as $association) { + $association->setInitialized(true); + $association->takeSnapshot(); } } @@ -2576,6 +2644,33 @@ public function loadCollection(PersistentCollection $collection): void $collection->setInitialized(true); } + /** + * Schedule this collection for batch loading at the end of the UnitOfWork + */ + private function scheduleCollectionForBatchLoading(PersistentCollection $collection, ClassMetadata $sourceClass): void + { + $mapping = $collection->getMapping(); + $name = $mapping['sourceEntity'] . '#' . $mapping['fieldName']; + + if (! isset($this->eagerLoadingCollections[$name])) { + $this->eagerLoadingCollections[$name] = [ + 'items' => [], + 'mapping' => $mapping, + ]; + } + + $owner = $collection->getOwner(); + assert($owner !== null); + + $id = $this->identifierFlattener->flattenIdentifier( + $sourceClass, + $sourceClass->getIdentifierValues($owner), + ); + $idHash = implode(' ', $id); + + $this->eagerLoadingCollections[$name]['items'][$idHash] = $collection; + } + /** * Gets the identity map of the UnitOfWork. * diff --git a/psalm.xml b/psalm.xml index bc88bcf3985..b2e7b723e42 100644 --- a/psalm.xml +++ b/psalm.xml @@ -31,6 +31,10 @@ + + + + diff --git a/tests/Doctrine/Tests/ORM/Functional/EagerFetchCollectionTest.php b/tests/Doctrine/Tests/ORM/Functional/EagerFetchCollectionTest.php new file mode 100644 index 00000000000..576bcfeabd2 --- /dev/null +++ b/tests/Doctrine/Tests/ORM/Functional/EagerFetchCollectionTest.php @@ -0,0 +1,136 @@ +createSchemaForModels(EagerFetchOwner::class, EagerFetchChild::class); + } + + public function testEagerFetchMode(): void + { + $owner = $this->createOwnerWithChildren(2); + $owner2 = $this->createOwnerWithChildren(3); + + $this->_em->flush(); + $this->_em->clear(); + + $owner = $this->_em->find(EagerFetchOwner::class, $owner->id); + + $afterQueryCount = count($this->getQueryLog()->queries); + $this->assertCount(2, $owner->children); + + $this->assertQueryCount($afterQueryCount, 'The $owner->children collection should already be initialized by find EagerFetchOwner before.'); + + $this->assertCount(3, $this->_em->find(EagerFetchOwner::class, $owner2->id)->children); + + $this->_em->clear(); + + $beforeQueryCount = count($this->getQueryLog()->queries); + $owners = $this->_em->getRepository(EagerFetchOwner::class)->findAll(); + + $this->assertQueryCount($beforeQueryCount + 2, 'the findAll() + 1 subselect loading both collections of the two returned $owners'); + + $this->assertCount(2, $owners[0]->children); + $this->assertCount(3, $owners[1]->children); + + $this->assertQueryCount($beforeQueryCount + 2, 'both collections are already initialized and counting them does not make a difference in total query count'); + } + + public function testEagerFetchModeWithDQL(): void + { + $owner = $this->createOwnerWithChildren(2); + $owner2 = $this->createOwnerWithChildren(3); + + $this->_em->flush(); + $this->_em->clear(); + + $query = $this->_em->createQuery('SELECT o FROM ' . EagerFetchOwner::class . ' o'); + $query->setFetchMode(EagerFetchOwner::class, 'children', ORM\ClassMetadata::FETCH_EAGER); + + $beforeQueryCount = count($this->getQueryLog()->queries); + $owners = $query->getResult(); + $afterQueryCount = count($this->getQueryLog()->queries); + + $this->assertEquals($beforeQueryCount + 2, $afterQueryCount); + + $owners[0]->children->count(); + $owners[1]->children->count(); + + $anotherQueryCount = count($this->getQueryLog()->queries); + + $this->assertEquals($anotherQueryCount, $afterQueryCount); + } + + public function testSubselectFetchJoinWithNotAllowed(): void + { + $this->expectException(QueryException::class); + $this->expectExceptionMessage('Associations with fetch-mode=EAGER may not be using WITH conditions'); + + $query = $this->_em->createQuery('SELECT o, c FROM ' . EagerFetchOwner::class . ' o JOIN o.children c WITH c.id = 1'); + $query->getResult(); + } + + protected function createOwnerWithChildren(int $children): EagerFetchOwner + { + $owner = new EagerFetchOwner(); + $this->_em->persist($owner); + + for ($i = 0; $i < $children; $i++) { + $child = new EagerFetchChild(); + $child->owner = $owner; + + $owner->children->add($child); + + $this->_em->persist($child); + } + + return $owner; + } +} + +#[ORM\Entity] +class EagerFetchOwner +{ + /** @var int */ + #[ORM\Column(type: 'integer')] + #[ORM\Id] + #[ORM\GeneratedValue] + public $id; + + #[ORM\OneToMany(mappedBy: 'owner', targetEntity: EagerFetchChild::class, fetch: 'EAGER')] + public Collection $children; + + public function __construct() + { + $this->children = new ArrayCollection(); + } +} + +#[ORM\Entity] +class EagerFetchChild +{ + /** @var int */ + #[ORM\Column(type: 'integer')] + #[ORM\Id] + #[ORM\GeneratedValue] + public $id; + + /** @var EagerFetchOwner */ + #[ORM\ManyToOne(targetEntity: EagerFetchOwner::class, inversedBy: 'children')] + public $owner; +} diff --git a/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC2350Test.php b/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC2350Test.php index c7fc322d2a4..c0b84c905a0 100644 --- a/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC2350Test.php +++ b/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC2350Test.php @@ -43,11 +43,11 @@ public function testEagerCollectionsAreOnlyRetrievedOnce(): void $this->getQueryLog()->reset()->enable(); $user = $this->_em->find(DDC2350User::class, $user->id); - $this->assertQueryCount(1); + $this->assertQueryCount(2); self::assertCount(2, $user->reportedBugs); - $this->assertQueryCount(1); + $this->assertQueryCount(2); } } diff --git a/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC440Test.php b/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC440Test.php index b65819f3ebc..b6b1e8bb542 100644 --- a/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC440Test.php +++ b/tests/Doctrine/Tests/ORM/Functional/Ticket/DDC440Test.php @@ -49,7 +49,7 @@ public function testOriginalEntityDataEmptyWhenProxyLoadedFromTwoAssociations(): $phone->setClient($client); $phone2 = new DDC440Phone(); - $phone->setId(2); + $phone2->setId(2); $phone2->setNumber('418 222-2222'); $phone2->setClient($client); diff --git a/tests/Doctrine/Tests/ORM/Functional/Ticket/GH10808Test.php b/tests/Doctrine/Tests/ORM/Functional/Ticket/GH10808Test.php index 833a034582e..0e893233442 100644 --- a/tests/Doctrine/Tests/ORM/Functional/Ticket/GH10808Test.php +++ b/tests/Doctrine/Tests/ORM/Functional/Ticket/GH10808Test.php @@ -40,8 +40,7 @@ public function testDQLDeferredEagerLoad(): void $query = $this->_em->createQuery( 'SELECT appointment from Doctrine\Tests\ORM\Functional\Ticket\GH10808Appointment appointment - JOIN appointment.child appointment_child - WITH appointment_child.id = 1', + JOIN appointment.child appointment_child', ); // By default, UnitOfWork::HINT_DEFEREAGERLOAD is set to 'true'