Skip to content

Sync 3.x #778

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
abnegate merged 16 commits into from
Jan 8, 2026
Merged

Sync 3.x #778

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
5a83504
added hashing for the key in postgres if more than 63
ArnabChatterjee20k Nov 17, 2025
1ee6436
linting
ArnabChatterjee20k Nov 17, 2025
68d47bf
changed hashing to use md5 instead of crc for collision issues
ArnabChatterjee20k Nov 18, 2025
ef4429e
renamed getIndexKey to getShortKey
ArnabChatterjee20k Nov 18, 2025
5da71b6
Merge pull request #761 from utopia-php/dat-923
abnegate Nov 18, 2025
737eee6
Add operator support for relationship arrays
abnegate Dec 5, 2025
af15066
Merge pull request #770 from utopia-php/feat-relationship-operators
abnegate Dec 8, 2025
9417bcb
Update limit message
abnegate Dec 16, 2025
c8c1b2f
Merge pull request #772 from utopia-php/chore-update-message
abnegate Dec 16, 2025
5e0008e
Add support for exists and notExists query types in Mongo
ArnabChatterjee20k Dec 31, 2025
086052d
added missing types to the query types
ArnabChatterjee20k Dec 31, 2025
b732828
updated exists and not exists method
ArnabChatterjee20k Jan 7, 2026
aa28027
linting
ArnabChatterjee20k Jan 7, 2026
60e1d32
Merge pull request #776 from utopia-php/query-exists
abnegate Jan 7, 2026
6d90ebe
Merge remote-tracking branch 'origin/3.x' into chore-sync-3.x
abnegate Jan 8, 2026
27d36f4
Merge remote-tracking branch 'origin/main' into chore-sync-3.x
abnegate Jan 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions src/Database/Adapter/Mongo.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ class Mongo extends Adapter
'$regex',
'$not',
'$nor',
'$exists',
];

protected Client $client;
Expand Down Expand Up @@ -2373,6 +2374,8 @@ protected function buildFilter(Query $query): array
$value = match ($query->getMethod()) {
Query::TYPE_IS_NULL,
Query::TYPE_IS_NOT_NULL => null,
Query::TYPE_EXISTS => true,
Query::TYPE_NOT_EXISTS => false,
default => $this->getQueryValue(
$query->getMethod(),
count($query->getValues()) > 1
Expand Down Expand Up @@ -2434,6 +2437,10 @@ protected function buildFilter(Query $query): array
$filter[$attribute] = ['$not' => $this->createSafeRegex($value, '^%s')];
} elseif ($operator === '$regex' && $query->getMethod() === Query::TYPE_NOT_ENDS_WITH) {
$filter[$attribute] = ['$not' => $this->createSafeRegex($value, '%s$')];
} elseif ($operator === '$exists') {
foreach ($query->getValues() as $attribute) {
$filter['$or'][] = [$attribute => [$operator => $value]];
}
} else {
$filter[$attribute][$operator] = $value;
}
Expand Down Expand Up @@ -2472,6 +2479,8 @@ protected function getQueryOperator(string $operator): string
Query::TYPE_NOT_ENDS_WITH => '$regex',
Query::TYPE_OR => '$or',
Query::TYPE_AND => '$and',
Query::TYPE_EXISTS,
Query::TYPE_NOT_EXISTS => '$exists',
default => throw new DatabaseException('Unknown operator:' . $operator . '. Must be one of ' . Query::TYPE_EQUAL . ', ' . Query::TYPE_NOT_EQUAL . ', ' . Query::TYPE_LESSER . ', ' . Query::TYPE_LESSER_EQUAL . ', ' . Query::TYPE_GREATER . ', ' . Query::TYPE_GREATER_EQUAL . ', ' . Query::TYPE_IS_NULL . ', ' . Query::TYPE_IS_NOT_NULL . ', ' . Query::TYPE_BETWEEN . ', ' . Query::TYPE_NOT_BETWEEN . ', ' . Query::TYPE_STARTS_WITH . ', ' . Query::TYPE_NOT_STARTS_WITH . ', ' . Query::TYPE_ENDS_WITH . ', ' . Query::TYPE_NOT_ENDS_WITH . ', ' . Query::TYPE_CONTAINS . ', ' . Query::TYPE_NOT_CONTAINS . ', ' . Query::TYPE_SEARCH . ', ' . Query::TYPE_NOT_SEARCH . ', ' . Query::TYPE_SELECT),
};
}
Expand Down
87 changes: 69 additions & 18 deletions src/Database/Adapter/Postgres.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
*/
class Postgres extends SQL
{
public const MAX_IDENTIFIER_NAME = 63;
/**
* @inheritDoc
*/
Expand Down Expand Up @@ -244,17 +245,24 @@ public function createCollection(string $name, array $attributes = [], array $in
";

if ($this->sharedTables) {
$uidIndex = $this->getShortKey("{$namespace}_{$this->tenant}_{$id}_uid");
$createdIndex = $this->getShortKey("{$namespace}_{$this->tenant}_{$id}_created");
$updatedIndex = $this->getShortKey("{$namespace}_{$this->tenant}_{$id}_updated");
$tenantIdIndex = $this->getShortKey("{$namespace}_{$this->tenant}_{$id}_tenant_id");
$collection .= "
CREATE UNIQUE INDEX \"{$namespace}_{$this->tenant}_{$id}_uid\" ON {$this->getSQLTable($id)} (\"_uid\" COLLATE utf8_ci_ai, \"_tenant\");
CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_created\" ON {$this->getSQLTable($id)} (_tenant, \"_createdAt\");
CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_updated\" ON {$this->getSQLTable($id)} (_tenant, \"_updatedAt\");
CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_tenant_id\" ON {$this->getSQLTable($id)} (_tenant, _id);
CREATE UNIQUE INDEX \"{$uidIndex}\" ON {$this->getSQLTable($id)} (\"_uid\" COLLATE utf8_ci_ai, \"_tenant\");
CREATE INDEX \"{$createdIndex}\" ON {$this->getSQLTable($id)} (_tenant, \"_createdAt\");
CREATE INDEX \"{$updatedIndex}\" ON {$this->getSQLTable($id)} (_tenant, \"_updatedAt\");
CREATE INDEX \"{$tenantIdIndex}\" ON {$this->getSQLTable($id)} (_tenant, _id);
";
} else {
$uidIndex = $this->getShortKey("{$namespace}_{$id}_uid");
$createdIndex = $this->getShortKey("{$namespace}_{$id}_created");
$updatedIndex = $this->getShortKey("{$namespace}_{$id}_updated");
$collection .= "
CREATE UNIQUE INDEX \"{$namespace}_{$id}_uid\" ON {$this->getSQLTable($id)} (\"_uid\" COLLATE utf8_ci_ai);
CREATE INDEX \"{$namespace}_{$id}_created\" ON {$this->getSQLTable($id)} (\"_createdAt\");
CREATE INDEX \"{$namespace}_{$id}_updated\" ON {$this->getSQLTable($id)} (\"_updatedAt\");
CREATE UNIQUE INDEX \"{$uidIndex}\" ON {$this->getSQLTable($id)} (\"_uid\" COLLATE utf8_ci_ai);
CREATE INDEX \"{$createdIndex}\" ON {$this->getSQLTable($id)} (\"_createdAt\");
CREATE INDEX \"{$updatedIndex}\" ON {$this->getSQLTable($id)} (\"_updatedAt\");
";
}

Expand All @@ -271,17 +279,21 @@ public function createCollection(string $name, array $attributes = [], array $in
";

if ($this->sharedTables) {
$uniquePermissionIndex = $this->getShortKey("{$namespace}_{$this->tenant}_{$id}_ukey");
$permissionIndex = $this->getShortKey("{$namespace}_{$this->tenant}_{$id}_permission");
$permissions .= "
CREATE UNIQUE INDEX \"{$namespace}_{$this->tenant}_{$id}_ukey\"
CREATE UNIQUE INDEX \"{$uniquePermissionIndex}\"
ON {$this->getSQLTable($id . '_perms')} USING btree (_tenant,_document,_type,_permission);
CREATE INDEX \"{$namespace}_{$this->tenant}_{$id}_permission\"
CREATE INDEX \"{$permissionIndex}\"
ON {$this->getSQLTable($id . '_perms')} USING btree (_tenant,_permission,_type);
";
} else {
$uniquePermissionIndex = $this->getShortKey("{$namespace}_{$id}_ukey");
$permissionIndex = $this->getShortKey("{$namespace}_{$id}_permission");
$permissions .= "
CREATE UNIQUE INDEX \"{$namespace}_{$id}_ukey\"
CREATE UNIQUE INDEX \"{$uniquePermissionIndex}\"
ON {$this->getSQLTable($id . '_perms')} USING btree (_document COLLATE utf8_ci_ai,_type,_permission);
CREATE INDEX \"{$namespace}_{$id}_permission\"
CREATE INDEX \"{$permissionIndex}\"
ON {$this->getSQLTable($id . '_perms')} USING btree (_permission,_type);
";
}
Expand Down Expand Up @@ -893,15 +905,15 @@ public function createIndex(string $collection, string $id, string $type, array
default => throw new DatabaseException('Unknown index type: ' . $type . '. Must be one of ' . Database::INDEX_KEY . ', ' . Database::INDEX_UNIQUE . ', ' . Database::INDEX_FULLTEXT . ', ' . Database::INDEX_SPATIAL . ', ' . Database::INDEX_OBJECT . ', ' . Database::INDEX_HNSW_EUCLIDEAN . ', ' . Database::INDEX_HNSW_COSINE . ', ' . Database::INDEX_HNSW_DOT),
};

$key = "\"{$this->getNamespace()}_{$this->tenant}_{$collection}_{$id}\"";
$keyName = $this->getShortKey("{$this->getNamespace()}_{$this->tenant}_{$collection}_{$id}");
$attributes = \implode(', ', $attributes);

if ($this->sharedTables && \in_array($type, [Database::INDEX_KEY, Database::INDEX_UNIQUE])) {
// Add tenant as first index column for best performance
$attributes = "_tenant, {$attributes}";
}

$sql = "CREATE {$sqlType} {$key} ON {$this->getSQLTable($collection)}";
$sql = "CREATE {$sqlType} \"{$keyName}\" ON {$this->getSQLTable($collection)}";

// Add USING clause for special index types
$sql .= match ($type) {
Expand Down Expand Up @@ -936,9 +948,9 @@ public function deleteIndex(string $collection, string $id): bool
$id = $this->filter($id);
$schemaName = $this->getDatabase();

$key = "\"{$this->getNamespace()}_{$this->tenant}_{$collection}_{$id}\"";
$keyName = $this->getShortKey("{$this->getNamespace()}_{$this->tenant}_{$collection}_{$id}");

$sql = "DROP INDEX IF EXISTS \"{$schemaName}\".{$key}";
$sql = "DROP INDEX IF EXISTS \"{$schemaName}\".\"{$keyName}\"";
$sql = $this->trigger(Database::EVENT_INDEX_DELETE, $sql);

return $this->execute($this->getPDO()
Expand All @@ -961,10 +973,11 @@ public function renameIndex(string $collection, string $old, string $new): bool
$namespace = $this->getNamespace();
$old = $this->filter($old);
$new = $this->filter($new);
$oldIndexName = "{$this->tenant}_{$collection}_{$old}";
$newIndexName = "{$namespace}_{$this->tenant}_{$collection}_{$new}";
$schema = $this->getDatabase();
$oldIndexName = $this->getShortKey("{$namespace}_{$this->tenant}_{$collection}_{$old}");
$newIndexName = $this->getShortKey("{$namespace}_{$this->tenant}_{$collection}_{$new}");

$sql = "ALTER INDEX {$this->getSQLTable($oldIndexName)} RENAME TO \"{$newIndexName}\"";
$sql = "ALTER INDEX \"{$schema}\".\"{$oldIndexName}\" RENAME TO \"{$newIndexName}\"";
$sql = $this->trigger(Database::EVENT_INDEX_RENAME, $sql);

return $this->execute($this->getPDO()
Expand Down Expand Up @@ -2743,4 +2756,42 @@ public function getSupportNonUtfCharacters(): bool
{
return false;
}

/**
* Ensure index key length stays within PostgreSQL's 63 character limit.
*
* @param string $key
* @return string
*/
protected function getShortKey(string $key): string
{
if (\strlen($key) <= self::MAX_IDENTIFIER_NAME) {
return $key;
}

$suffix = '';
$separatorPosition = strrpos($key, '_');
if ($separatorPosition !== false) {
$suffix = substr($key, $separatorPosition + 1);
}

$hash = md5($key);

if ($suffix !== '') {
$hashedKey = "{$hash}_{$suffix}";
if (\strlen($hashedKey) <= self::MAX_IDENTIFIER_NAME) {
return $hashedKey;
}
}

return substr($hash, 0, self::MAX_IDENTIFIER_NAME);
}

protected function getSQLTable(string $name): string
{
$table = "{$this->getNamespace()}_{$this->filter($name)}";
$table = $this->getShortKey($table);

return "{$this->quote($this->getDatabase())}.{$this->quote($table)}";
}
}
3 changes: 3 additions & 0 deletions src/Database/Adapter/SQL.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1797,6 +1797,9 @@ protected function getSQLOperator(string $method): string
case Query::TYPE_VECTOR_COSINE:
case Query::TYPE_VECTOR_EUCLIDEAN:
throw new DatabaseException('Vector queries are not supported by this database');
case Query::TYPE_EXISTS:
case Query::TYPE_NOT_EXISTS:
throw new DatabaseException('Exists queries are not supported by this database');
default:
throw new DatabaseException('Unknown method: ' . $method);
}
Expand Down
84 changes: 82 additions & 2 deletions src/Database/Database.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3003,14 +3003,14 @@ public function checkAttribute(Document $collection, Document $attribute): bool
$this->adapter->getLimitForAttributes() > 0 &&
$this->adapter->getCountOfAttributes($collection) > $this->adapter->getLimitForAttributes()
) {
throw new LimitException('Column limit reached. Cannot create new attribute.');
throw new LimitException('Column limit reached. Cannot create new attribute. Current attribute count is ' . $this->adapter->getCountOfAttributes($collection) . ' but the maximum is ' . $this->adapter->getLimitForAttributes() . '. Remove some attributes to free up space.');
}

if (
$this->adapter->getDocumentSizeLimit() > 0 &&
$this->adapter->getAttributeWidth($collection) >= $this->adapter->getDocumentSizeLimit()
) {
throw new LimitException('Row width limit reached. Cannot create new attribute.');
throw new LimitException('Row width limit reached. Cannot create new attribute. Current row width is ' . $this->adapter->getAttributeWidth($collection) . ' bytes but the maximum is ' . $this->adapter->getDocumentSizeLimit() . ' bytes. Reduce the size of existing attributes or remove some attributes to free up space.');
}

return true;
Expand Down Expand Up @@ -5671,6 +5671,11 @@ public function updateDocument(string $collection, string $id, Document $documen
break;
}

if (Operator::isOperator($value)) {
$shouldUpdate = true;
break;
}

if (!\is_array($value) || !\array_is_list($value)) {
throw new RelationshipException('Invalid relationship value. Must be either an array of documents or document IDs, ' . \gettype($value) . ' given.');
}
Expand Down Expand Up @@ -6079,6 +6084,24 @@ private function updateDocumentRelationships(Document $collection, Document $old
$twoWayKey = (string)$relationship['options']['twoWayKey'];
$side = (string)$relationship['options']['side'];

if (Operator::isOperator($value)) {
$operator = $value;
if ($operator->isArrayOperation()) {
$existingIds = [];
if (\is_array($oldValue)) {
$existingIds = \array_map(function ($item) {
if ($item instanceof Document) {
return $item->getId();
}
return $item;
}, $oldValue);
}

$value = $this->applyRelationshipOperator($operator, $existingIds);
$document->setAttribute($key, $value);
}
}

if ($oldValue == $value) {
if (
($relationType === Database::RELATION_ONE_TO_ONE
Expand Down Expand Up @@ -6438,6 +6461,63 @@ private function getJunctionCollection(Document $collection, Document $relatedCo
: '_' . $relatedCollection->getSequence() . '_' . $collection->getSequence();
}

/**
* Apply an operator to a relationship array of IDs
*
* @param Operator $operator
* @param array<string> $existingIds
* @return array<string|Document>
*/
private function applyRelationshipOperator(Operator $operator, array $existingIds): array
{
$method = $operator->getMethod();
$values = $operator->getValues();

// Extract IDs from operator values (could be strings or Documents)
$valueIds = \array_filter(\array_map(fn ($item) => $item instanceof Document ? $item->getId() : (\is_string($item) ? $item : null), $values));

switch ($method) {
case Operator::TYPE_ARRAY_APPEND:
return \array_values(\array_merge($existingIds, $valueIds));

case Operator::TYPE_ARRAY_PREPEND:
return \array_values(\array_merge($valueIds, $existingIds));

case Operator::TYPE_ARRAY_INSERT:
$index = $values[0] ?? 0;
$item = $values[1] ?? null;
$itemId = $item instanceof Document ? $item->getId() : (\is_string($item) ? $item : null);
if ($itemId !== null) {
\array_splice($existingIds, $index, 0, [$itemId]);
}
return \array_values($existingIds);

case Operator::TYPE_ARRAY_REMOVE:
$toRemove = $values[0] ?? null;
if (\is_array($toRemove)) {
$toRemoveIds = \array_filter(\array_map(fn ($item) => $item instanceof Document ? $item->getId() : (\is_string($item) ? $item : null), $toRemove));
return \array_values(\array_diff($existingIds, $toRemoveIds));
}
$toRemoveId = $toRemove instanceof Document ? $toRemove->getId() : (\is_string($toRemove) ? $toRemove : null);
if ($toRemoveId !== null) {
return \array_values(\array_diff($existingIds, [$toRemoveId]));
}
return $existingIds;

case Operator::TYPE_ARRAY_UNIQUE:
return \array_values(\array_unique($existingIds));

case Operator::TYPE_ARRAY_INTERSECT:
return \array_values(\array_intersect($existingIds, $valueIds));

case Operator::TYPE_ARRAY_DIFF:
return \array_values(\array_diff($existingIds, $valueIds));

default:
return $existingIds;
}
}

/**
* Create or update a document.
*
Expand Down
30 changes: 29 additions & 1 deletion src/Database/Query.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@ class Query
public const TYPE_NOT_STARTS_WITH = 'notStartsWith';
public const TYPE_ENDS_WITH = 'endsWith';
public const TYPE_NOT_ENDS_WITH = 'notEndsWith';
public const TYPE_EXISTS = 'exists';
public const TYPE_NOT_EXISTS = 'notExists';

// Spatial methods
public const TYPE_CROSSES = 'crosses';
Expand Down Expand Up @@ -99,6 +101,8 @@ class Query
self::TYPE_VECTOR_DOT,
self::TYPE_VECTOR_COSINE,
self::TYPE_VECTOR_EUCLIDEAN,
self::TYPE_EXISTS,
self::TYPE_NOT_EXISTS,
self::TYPE_SELECT,
self::TYPE_ORDER_DESC,
self::TYPE_ORDER_ASC,
Expand Down Expand Up @@ -294,7 +298,9 @@ public static function isMethod(string $value): bool
self::TYPE_SELECT,
self::TYPE_VECTOR_DOT,
self::TYPE_VECTOR_COSINE,
self::TYPE_VECTOR_EUCLIDEAN => true,
self::TYPE_VECTOR_EUCLIDEAN,
self::TYPE_EXISTS,
self::TYPE_NOT_EXISTS => true,
default => false,
};
}
Expand Down Expand Up @@ -1178,4 +1184,26 @@ public static function vectorEuclidean(string $attribute, array $vector): self
{
return new self(self::TYPE_VECTOR_EUCLIDEAN, $attribute, [$vector]);
}

/**
* Helper method to create Query with exists method
*
* @param array<string> $attributes
* @return Query
*/
public static function exists(array $attributes): self
{
return new self(self::TYPE_EXISTS, '', $attributes);
}

/**
* Helper method to create Query with notExists method
*
* @param string|int|float|bool|array<mixed,mixed> $attribute
* @return Query
*/
public static function notExists(string|int|float|bool|array $attribute): self
{
return new self(self::TYPE_NOT_EXISTS, '', is_array($attribute) ? $attribute : [$attribute]);
}
}
Loading