From 5f637cbd2beee66c3a4318b36a9d5996557b1301 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Fri, 9 Jan 2026 17:57:45 +0530 Subject: [PATCH 01/16] feat: add support for TTL indexes in database adapters and validation --- src/Database/Adapter.php | 13 +- src/Database/Adapter/MariaDB.php | 2 +- src/Database/Adapter/Mongo.php | 27 +- src/Database/Adapter/Pool.php | 2 +- src/Database/Adapter/Postgres.php | 7 +- src/Database/Adapter/SQLite.php | 5 +- src/Database/Database.php | 10 +- src/Database/Mirror.php | 7 +- src/Database/Validator/Index.php | 50 ++- tests/e2e/Adapter/Scopes/SchemalessTests.php | 324 +++++++++++++++++++ tests/unit/Validator/IndexTest.php | 179 ++++++++++ 11 files changed, 611 insertions(+), 15 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 49a33e403..603c18be5 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -666,10 +666,12 @@ abstract public function renameIndex(string $collection, string $old, string $ne * @param array $lengths * @param array $orders * @param array $indexAttributeTypes + * @param array $collation + * @param int $ttl * * @return bool */ - abstract public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool; + abstract public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool; /** * Delete Index @@ -1483,4 +1485,13 @@ public function getSupportForRegex(): bool { return $this->getSupportForPCRERegex() || $this->getSupportForPOSIXRegex(); } + + /** + * Are ttl indexes supported? + * + * @return bool + */ + public function getSupportTTLIndexes(): bool { + return false; + } } diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index b8110b039..ef44171c6 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -715,7 +715,7 @@ public function renameIndex(string $collection, string $old, string $new): bool * @return bool * @throws DatabaseException */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool { $metadataCollection = new Document(['$id' => Database::METADATA]); $collection = $this->getDocument($metadataCollection, $collection); diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index fdcdd80c2..e3441f92a 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -508,6 +508,9 @@ public function createCollection(string $name, array $attributes = [], array $in $order = $this->getOrder($this->filter($orders[$j] ?? Database::ORDER_ASC)); $unique = true; break; + case Database::INDEX_TTL: + $order = $this->getOrder($this->filter($orders[$j] ?? Database::ORDER_ASC)); + break; default: // index not supported return false; @@ -526,6 +529,14 @@ public function createCollection(string $name, array $attributes = [], array $in $newIndexes[$i]['default_language'] = 'none'; } + // Handle TTL indexes + if ($index->getAttribute('type') === Database::INDEX_TTL) { + $ttl = $index->getAttribute('ttl', 0); + if ($ttl > 0) { + $newIndexes[$i]['expireAfterSeconds'] = $ttl; + } + } + // Add partial filter for indexes to avoid indexing null values if (in_array($index->getAttribute('type'), [ Database::INDEX_UNIQUE, @@ -901,10 +912,11 @@ public function deleteRelationship( * @param array $orders * @param array $indexAttributeTypes * @param array $collation + * @param int $ttl * @return bool * @throws Exception */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool { $name = $this->getNamespace() . '_' . $this->filter($collection); $id = $this->filter($id); @@ -933,6 +945,8 @@ public function createIndex(string $collection, string $id, string $type, array case Database::INDEX_UNIQUE: $indexes['unique'] = true; break; + case Database::INDEX_TTL: + break; default: return false; } @@ -961,6 +975,11 @@ public function createIndex(string $collection, string $id, string $type, array $indexes['default_language'] = 'none'; } + // Handle TTL indexes + if ($type === Database::INDEX_TTL && $ttl > 0) { + $indexes['expireAfterSeconds'] = $ttl; + } + // Add partial filter for indexes to avoid indexing null values if (in_array($type, [Database::INDEX_UNIQUE, Database::INDEX_KEY])) { $partialFilter = []; @@ -1073,7 +1092,7 @@ public function renameIndex(string $collection, string $old, string $new): bool try { $deletedindex = $this->deleteIndex($collection, $old); - $createdindex = $this->createIndex($collection, $new, $index['type'], $index['attributes'], $index['lengths'] ?? [], $index['orders'] ?? [], $indexAttributeTypes); + $createdindex = $this->createIndex($collection, $new, $index['type'], $index['attributes'], $index['lengths'] ?? [], $index['orders'] ?? [], $indexAttributeTypes, [], $index['ttl'] ?? 0); } catch (\Exception $e) { throw $this->processException($e); } @@ -3395,4 +3414,8 @@ public function getSupportForTrigramIndex(): bool { return false; } + + public function getSupportTTLIndexes(): bool { + return true; + } } diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index d70a836ea..e6bf77681 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -210,7 +210,7 @@ public function renameIndex(string $collection, string $old, string $new): bool return $this->delegate(__FUNCTION__, \func_get_args()); } - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool { return $this->delegate(__FUNCTION__, \func_get_args()); } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 050180a0a..38c3c3a26 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -319,6 +319,7 @@ public function createCollection(string $name, array $attributes = [], array $in } } $indexOrders = $index->getAttribute('orders', []); + $indexTtl = $index->getAttribute('ttl', 0); if ($indexType === Database::INDEX_SPATIAL && count($indexOrders)) { throw new DatabaseException('Spatial indexes with explicit orders are not supported. Remove the orders to create this index.'); } @@ -329,7 +330,9 @@ public function createCollection(string $name, array $attributes = [], array $in $indexAttributes, [], $indexOrders, - $indexAttributesWithType + $indexAttributesWithType, + [], + $indexTtl ); } } catch (PDOException $e) { @@ -876,7 +879,7 @@ public function deleteRelationship( * @return bool */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool { $collection = $this->filter($collection); $id = $this->filter($id); diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 948070654..538b93a68 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -216,8 +216,9 @@ public function createCollection(string $name, array $attributes = [], array $in $indexAttributes = $index->getAttribute('attributes', []); $indexLengths = $index->getAttribute('lengths', []); $indexOrders = $index->getAttribute('orders', []); + $indexTtl = $index->getAttribute('ttl', 0); - $this->createIndex($id, $indexId, $indexType, $indexAttributes, $indexLengths, $indexOrders); + $this->createIndex($id, $indexId, $indexType, $indexAttributes, $indexLengths, $indexOrders, [], [], $indexTtl); } $this->createIndex("{$id}_perms", '_index_1', Database::INDEX_UNIQUE, ['_document', '_type', '_permission'], [], []); @@ -455,7 +456,7 @@ public function renameIndex(string $collection, string $old, string $new): bool * @throws Exception * @throws PDOException */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool { $name = $this->filter($collection); $id = $this->filter($id); diff --git a/src/Database/Database.php b/src/Database/Database.php index a2bc2da55..f12795bf7 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -86,6 +86,7 @@ class Database public const INDEX_HNSW_COSINE = 'hnsw_cosine'; public const INDEX_HNSW_DOT = 'hnsw_dot'; public const INDEX_TRIGRAM = 'trigram'; + public const INDEX_TTL = 'ttl'; // Max limits public const MAX_INT = 2147483647; @@ -1648,6 +1649,7 @@ public function createCollection(string $id, array $attributes = [], array $inde $this->adapter->getSupportForIndex(), $this->adapter->getSupportForUniqueIndex(), $this->adapter->getSupportForFulltextIndex(), + $this->adapter->getSupportTTLIndexes() ); foreach ($indexes as $index) { if (!$validator->isValid($index)) { @@ -2798,6 +2800,7 @@ public function updateAttribute(string $collection, string $id, ?string $type = $this->adapter->getSupportForIndex(), $this->adapter->getSupportForUniqueIndex(), $this->adapter->getSupportForFulltextIndex(), + $this->adapter->getSupportTTLIndexes() ); foreach ($indexes as $index) { @@ -3603,6 +3606,7 @@ public function renameIndex(string $collection, string $old, string $new): bool * @param array $attributes * @param array $lengths * @param array $orders + * @param int $ttl * * @return bool * @throws AuthorizationException @@ -3613,7 +3617,7 @@ public function renameIndex(string $collection, string $old, string $new): bool * @throws StructureException * @throws Exception */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = [], int $ttl = 0): bool { if (empty($attributes)) { throw new DatabaseException('Missing attributes'); @@ -3671,6 +3675,7 @@ public function createIndex(string $collection, string $id, string $type, array 'attributes' => $attributes, 'lengths' => $lengths, 'orders' => $orders, + 'ttl' => $ttl ]); if ($this->validate) { @@ -3693,6 +3698,7 @@ public function createIndex(string $collection, string $id, string $type, array $this->adapter->getSupportForIndex(), $this->adapter->getSupportForUniqueIndex(), $this->adapter->getSupportForFulltextIndex(), + $this->adapter->getSupportTTLIndexes() ); if (!$validator->isValid($index)) { throw new IndexException($validator->getDescription()); @@ -3702,7 +3708,7 @@ public function createIndex(string $collection, string $id, string $type, array $collection->setAttribute('indexes', $index, Document::SET_TYPE_APPEND); try { - $created = $this->adapter->createIndex($collection->getId(), $id, $type, $attributes, $lengths, $orders, $indexAttributesWithTypes); + $created = $this->adapter->createIndex($collection->getId(), $id, $type, $attributes, $lengths, $orders, $indexAttributesWithTypes, [], $ttl); if (!$created) { throw new DatabaseException('Failed to create index'); diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php index d9a6d09df..3c65a88a9 100644 --- a/src/Database/Mirror.php +++ b/src/Database/Mirror.php @@ -469,9 +469,9 @@ public function deleteAttribute(string $collection, string $id): bool return $result; } - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = [], int $ttl = 0): bool { - $result = $this->source->createIndex($collection, $id, $type, $attributes, $lengths, $orders); + $result = $this->source->createIndex($collection, $id, $type, $attributes, $lengths, $orders, $ttl); if ($this->destination === null) { return $result; @@ -502,7 +502,8 @@ public function createIndex(string $collection, string $id, string $type, array $document->getAttribute('type'), $document->getAttribute('attributes'), $document->getAttribute('lengths'), - $document->getAttribute('orders') + $document->getAttribute('orders'), + $document->getAttribute('ttl', 0) ); } catch (\Throwable $err) { $this->logError('createIndex', $err); diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php index e2fc70a0b..243cde156 100644 --- a/src/Database/Validator/Index.php +++ b/src/Database/Validator/Index.php @@ -54,6 +54,7 @@ public function __construct( protected bool $supportForKeyIndexes = true, protected bool $supportForUniqueIndexes = true, protected bool $supportForFulltextIndexes = true, + protected bool $supportForTTLIndexes = false, ) { foreach ($attributes as $attribute) { $key = \strtolower($attribute->getAttribute('key', $attribute->getAttribute('$id'))); @@ -156,6 +157,9 @@ public function isValid($value): bool if (!$this->checkKeyUniqueFulltextSupport($value)) { return false; } + if (!$this->checkTTLIndexes($value)) { + return false; + } return true; } @@ -222,6 +226,13 @@ public function checkValidIndex(Document $index): bool } break; + case Database::INDEX_TTL: + if (!$this->supportForTTLIndexes) { + $this->message = 'TTL indexes are not supported'; + return false; + } + break; + default: $this->message = 'Unknown index type: ' . $type . '. Must be one of ' . Database::INDEX_KEY . ', ' . Database::INDEX_UNIQUE . ', ' . Database::INDEX_FULLTEXT . ', ' . Database::INDEX_SPATIAL . ', ' . Database::INDEX_OBJECT . ', ' . Database::INDEX_HNSW_EUCLIDEAN . ', ' . Database::INDEX_HNSW_COSINE . ', ' . Database::INDEX_HNSW_DOT . ', '.Database::INDEX_TRIGRAM; return false; @@ -673,7 +684,7 @@ public function checkIdenticalIndexes(Document $index): bool if ($attributesMatch && $ordersMatch) { // Allow fulltext + key/unique combinations (different purposes) - $regularTypes = [Database::INDEX_KEY, Database::INDEX_UNIQUE]; + $regularTypes = [Database::INDEX_KEY, Database::INDEX_UNIQUE, Database::INDEX_TTL]; $isRegularIndex = \in_array($indexType, $regularTypes); $isRegularExisting = \in_array($existingType, $regularTypes); @@ -729,4 +740,41 @@ public function checkObjectIndexes(Document $index): bool return true; } + + public function checkTTLIndexes(Document $index): bool{ + $type = $index->getAttribute('type'); + + $attributes = $index->getAttribute('attributes', []); + $orders = $index->getAttribute('orders', []); + $ttl = $index->getAttribute('ttl', 0); + if ($type !== Database::INDEX_TTL) { + return true; + } + + if (count($attributes) !== 1) { + $this->message = 'TTL index can be created on a single object attribute'; + return false; + } + + if (empty($orders)) { + $this->message = 'TTL index need explicit orders. Add the orders to create this index.'; + return false; + } + + $attributeName = $attributes[0] ?? ''; + $attribute = $this->attributes[\strtolower($attributeName)] ?? new Document(); + $attributeType = $attribute->getAttribute('type', ''); + + if ($attributeType !== Database::VAR_DATETIME) { + $this->message = 'Object index can only be created on datetime attributes. Attribute "' . $attributeName . '" is of type "' . $attributeType . '"'; + return false; + } + + if($ttl <= 0){ + $this->message = 'TTL must be atleast 1 second'; + return false; + } + + return true; + } } diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php index 856d08263..3c5b0a76a 100644 --- a/tests/e2e/Adapter/Scopes/SchemalessTests.php +++ b/tests/e2e/Adapter/Scopes/SchemalessTests.php @@ -6,6 +6,7 @@ use Throwable; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Authorization as AuthorizationException; use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\Limit as LimitException; @@ -1865,4 +1866,327 @@ public function testSchemalessNestedObjectAttributeQueries(): void $database->deleteCollection($col); } + + public function testSchemalessTTLIndexes(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + // Only run for MongoDB adapter which supports TTL indexes + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl'); + $database->createCollection($col); + + // Create datetime attribute for TTL index + $database->createAttribute($col, 'expiresAt', Database::VAR_DATETIME, 0, false); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + // Test 1: Create valid TTL index with valid TTL value + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_valid', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 3600 // 1 hour TTL + ) + ); + + // Verify index was created and stored in metadata + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(1, $indexes); + $ttlIndex = $indexes[0]; + $this->assertEquals('idx_ttl_valid', $ttlIndex->getId()); + $this->assertEquals(Database::INDEX_TTL, $ttlIndex->getAttribute('type')); + $this->assertEquals(3600, $ttlIndex->getAttribute('ttl')); + + // Test 2: Create documents with expiresAt field + $now = new \DateTime(); + $future1 = (clone $now)->modify('+2 hours'); + $future2 = (clone $now)->modify('+1 hour'); + $past = (clone $now)->modify('-1 hour'); + + $doc1 = $database->createDocument($col, new Document([ + '$id' => 'doc1', + '$permissions' => $permissions, + 'expiresAt' => $future1->format(\DateTime::ATOM), + 'data' => 'will expire in 2 hours' + ])); + + $doc2 = $database->createDocument($col, new Document([ + '$id' => 'doc2', + '$permissions' => $permissions, + 'expiresAt' => $future2->format(\DateTime::ATOM), + 'data' => 'will expire in 1 hour' + ])); + + $doc3 = $database->createDocument($col, new Document([ + '$id' => 'doc3', + '$permissions' => $permissions, + 'expiresAt' => $past->format(\DateTime::ATOM), + 'data' => 'already expired' + ])); + + // Verify documents were created + $this->assertEquals('doc1', $doc1->getId()); + $this->assertEquals('doc2', $doc2->getId()); + $this->assertEquals('doc3', $doc3->getId()); + + // Test 3: Delete the first TTL index and create a new one with minimum valid TTL (1 second) + // MongoDB only allows one TTL index per attribute, so we need to delete the existing one first + $this->assertTrue($database->deleteIndex($col, 'idx_ttl_valid')); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_min', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 1 // Minimum TTL + ) + ); + + // Test 4: Try to create TTL index with TTL = 0 (should fail) + try { + $database->createIndex( + $col, + 'idx_ttl_zero', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 0 + ); + $this->fail('Expected exception for TTL = 0'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('TTL must be atleast 1 second', $e->getMessage()); + } + + // Test 5: Try to create TTL index with negative TTL (should fail) + try { + $database->createIndex( + $col, + 'idx_ttl_negative', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + -100 + ); + $this->fail('Expected exception for negative TTL'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('TTL must be atleast 1 second', $e->getMessage()); + } + + // Test 6: Create TTL index via createCollection with indexes parameter + $col2 = uniqid('sl_ttl_collection'); + + // Create attribute document for the collection + $expiresAtAttr = new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]); + + $ttlIndexDoc = new Document([ + '$id' => ID::custom('idx_ttl_collection'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200 // 2 hours + ]); + + $database->createCollection($col2, [$expiresAtAttr], [$ttlIndexDoc]); + + // Verify TTL index was created via createCollection + $collection2 = $database->getCollection($col2); + $indexes2 = $collection2->getAttribute('indexes'); + $this->assertCount(1, $indexes2); + $ttlIndex2 = $indexes2[0]; + $this->assertEquals('idx_ttl_collection', $ttlIndex2->getId()); + $this->assertEquals(7200, $ttlIndex2->getAttribute('ttl')); + + // Cleanup + $database->deleteCollection($col); + $database->deleteCollection($col2); + } + + public function testSchemalessTTLIndexDuplicatePrevention(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + // Only run for MongoDB adapter which supports TTL indexes + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl_dup'); + $database->createCollection($col); + + // Create datetime attribute for TTL index + $database->createAttribute($col, 'expiresAt', Database::VAR_DATETIME, 0, false); + $database->createAttribute($col, 'deletedAt', Database::VAR_DATETIME, 0, false); + + // Test 1: Create first TTL index on expiresAt + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expires', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 3600 // 1 hour + ) + ); + + // Test 2: Try to create another TTL index on the same attribute (should fail) + try { + $database->createIndex( + $col, + 'idx_ttl_expires_duplicate', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 7200 // 2 hours + ); + $this->fail('Expected exception for duplicate TTL index on same attribute'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('A TTL index already exists on attribute', $e->getMessage()); + } + + // Test 3: Create TTL index on different attribute (should succeed) + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_deleted', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 86400 // 24 hours + ) + ); + + // Verify both indexes exist + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(2, $indexes); + + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); + $this->assertContains('idx_ttl_expires', $indexIds); + $this->assertContains('idx_ttl_deleted', $indexIds); + + // Test 4: Try to create another TTL index on deletedAt (should fail) + try { + $database->createIndex( + $col, + 'idx_ttl_deleted_duplicate', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 172800 // 48 hours + ); + $this->fail('Expected exception for duplicate TTL index on same attribute'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('A TTL index already exists on attribute', $e->getMessage()); + } + + // Test 5: Delete first TTL index and create a new one on same attribute (should succeed) + $this->assertTrue($database->deleteIndex($col, 'idx_ttl_expires')); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expires_new', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 1800 // 30 minutes + ) + ); + + // Verify the new index replaced the old one + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(2, $indexes); + + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); + $this->assertNotContains('idx_ttl_expires', $indexIds); + $this->assertContains('idx_ttl_expires_new', $indexIds); + $this->assertContains('idx_ttl_deleted', $indexIds); + + // Test 6: Try to create TTL index via createCollection with duplicate (should fail validation) + $col3 = uniqid('sl_ttl_dup_collection'); + + $expiresAtAttr = new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]); + + $ttlIndex1 = new Document([ + '$id' => ID::custom('idx_ttl_1'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 3600 + ]); + + $ttlIndex2 = new Document([ + '$id' => ID::custom('idx_ttl_2'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200 + ]); + + try { + $database->createCollection($col3, [$expiresAtAttr], [$ttlIndex1, $ttlIndex2]); + $this->fail('Expected exception for duplicate TTL indexes in createCollection'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('A TTL index already exists on attribute', $e->getMessage()); + } + + // Cleanup + $database->deleteCollection($col); + } } diff --git a/tests/unit/Validator/IndexTest.php b/tests/unit/Validator/IndexTest.php index 5dfe80e4e..b8bfee056 100644 --- a/tests/unit/Validator/IndexTest.php +++ b/tests/unit/Validator/IndexTest.php @@ -596,4 +596,183 @@ public function testTrigramIndexValidation(): void $this->assertFalse($validatorNoSupport->isValid($validIndex)); $this->assertEquals('Trigram indexes are not supported', $validatorNoSupport->getDescription()); } + + /** + * @throws Exception + */ + public function testTTLIndexValidation(): void + { + $collection = new Document([ + '$id' => ID::custom('test'), + 'name' => 'test', + 'attributes' => [ + new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]), + new Document([ + '$id' => ID::custom('name'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ]), + ], + 'indexes' => [] + ]); + + // Validator with supportForTTLIndexes enabled + $validator = new Index( + $collection->getAttribute('attributes'), + $collection->getAttribute('indexes', []), + 768, + [], + false, // supportForArrayIndexes + false, // supportForSpatialIndexNull + false, // supportForSpatialIndexOrder + false, // supportForVectorIndexes + true, // supportForAttributes + true, // supportForMultipleFulltextIndexes + true, // supportForIdenticalIndexes + false, // supportForObjectIndexes + false, // supportForTrigramIndexes + false, // supportForSpatialIndexes + true, // supportForKeyIndexes + true, // supportForUniqueIndexes + true, // supportForFulltextIndexes + true // supportForTTLIndexes + ); + + // Valid: TTL index on single datetime attribute with valid TTL + $validIndex = new Document([ + '$id' => ID::custom('idx_ttl_valid'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 3600, + ]); + $this->assertTrue($validator->isValid($validIndex)); + + // Invalid: TTL index with TTL = 0 + $invalidIndexZero = new Document([ + '$id' => ID::custom('idx_ttl_zero'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 0, + ]); + $this->assertFalse($validator->isValid($invalidIndexZero)); + $this->assertEquals('TTL must be atleast 1 second', $validator->getDescription()); + + // Invalid: TTL index with TTL < 0 + $invalidIndexNegative = new Document([ + '$id' => ID::custom('idx_ttl_negative'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => -100, + ]); + $this->assertFalse($validator->isValid($invalidIndexNegative)); + $this->assertEquals('TTL must be atleast 1 second', $validator->getDescription()); + + // Invalid: TTL index on non-datetime attribute + $invalidIndexType = new Document([ + '$id' => ID::custom('idx_ttl_invalid_type'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['name'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 3600, + ]); + $this->assertFalse($validator->isValid($invalidIndexType)); + $this->assertStringContainsString('Object index can only be created on datetime attributes', $validator->getDescription()); + + // Invalid: TTL index on multiple attributes + $invalidIndexMulti = new Document([ + '$id' => ID::custom('idx_ttl_multi'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt', 'name'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], + 'ttl' => 3600, + ]); + $this->assertFalse($validator->isValid($invalidIndexMulti)); + $this->assertStringContainsString('TTL index can be created on a single object attribute', $validator->getDescription()); + + // Invalid: TTL index without orders + $invalidIndexNoOrders = new Document([ + '$id' => ID::custom('idx_ttl_no_orders'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [], + 'ttl' => 3600, + ]); + $this->assertFalse($validator->isValid($invalidIndexNoOrders)); + $this->assertEquals('TTL index need explicit orders. Add the orders to create this index.', $validator->getDescription()); + + // Valid: TTL index with minimum valid TTL (1 second) + $validIndexMin = new Document([ + '$id' => ID::custom('idx_ttl_min'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 1, + ]); + $this->assertTrue($validator->isValid($validIndexMin)); + + // Invalid: TTL index on same attribute when another TTL index already exists + $collection->setAttribute('indexes', $validIndex, Document::SET_TYPE_APPEND); + $validatorWithExisting = new Index( + $collection->getAttribute('attributes'), + $collection->getAttribute('indexes', []), + 768, + [], + false, // supportForArrayIndexes + false, // supportForSpatialIndexNull + false, // supportForSpatialIndexOrder + false, // supportForVectorIndexes + true, // supportForAttributes + true, // supportForMultipleFulltextIndexes + true, // supportForIdenticalIndexes + false, // supportForObjectIndexes + false, // supportForTrigramIndexes + false, // supportForSpatialIndexes + true, // supportForKeyIndexes + true, // supportForUniqueIndexes + true, // supportForFulltextIndexes + true // supportForTTLIndexes + ); + + $duplicateTTLIndex = new Document([ + '$id' => ID::custom('idx_ttl_duplicate'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200, + ]); + $this->assertFalse($validatorWithExisting->isValid($duplicateTTLIndex)); + $this->assertStringContainsString('A TTL index already exists on attribute', $validatorWithExisting->getDescription()); + + // Validator with supportForTrigramIndexes disabled should reject TTL + $validatorNoSupport = new Index($collection->getAttribute('attributes'), $collection->getAttribute('indexes', []), 768, [], false, false, false, false, false, false, false, false, false); + $this->assertFalse($validatorNoSupport->isValid($validIndex)); + $this->assertEquals('TTL indexes are not supported', $validatorNoSupport->getDescription()); + } } From 321a4c53a11d72af961d7eae28819fc3351504d5 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 12 Jan 2026 13:40:47 +0530 Subject: [PATCH 02/16] refactor: standardize method formatting and improve TTL index validation tests --- src/Database/Adapter.php | 3 +- src/Database/Adapter/Mongo.php | 3 +- src/Database/Database.php | 1 - src/Database/Validator/Index.php | 28 +- tests/e2e/Adapter/Scopes/DocumentTests.php | 261 +++++++++++++++++++ tests/e2e/Adapter/Scopes/SchemalessTests.php | 83 +----- tests/unit/Validator/IndexTest.php | 4 +- 7 files changed, 300 insertions(+), 83 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 603c18be5..37772e1b2 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -1491,7 +1491,8 @@ public function getSupportForRegex(): bool * * @return bool */ - public function getSupportTTLIndexes(): bool { + public function getSupportTTLIndexes(): bool + { return false; } } diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index e3441f92a..cf3d28189 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -3415,7 +3415,8 @@ public function getSupportForTrigramIndex(): bool return false; } - public function getSupportTTLIndexes(): bool { + public function getSupportTTLIndexes(): bool + { return true; } } diff --git a/src/Database/Database.php b/src/Database/Database.php index f12795bf7..65526c348 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -3624,7 +3624,6 @@ public function createIndex(string $collection, string $id, string $type, array } $collection = $this->silent(fn () => $this->getCollection($collection)); - // index IDs are case-insensitive $indexes = $collection->getAttribute('indexes', []); diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php index 243cde156..5226d3300 100644 --- a/src/Database/Validator/Index.php +++ b/src/Database/Validator/Index.php @@ -232,7 +232,7 @@ public function checkValidIndex(Document $index): bool return false; } break; - + default: $this->message = 'Unknown index type: ' . $type . '. Must be one of ' . Database::INDEX_KEY . ', ' . Database::INDEX_UNIQUE . ', ' . Database::INDEX_FULLTEXT . ', ' . Database::INDEX_SPATIAL . ', ' . Database::INDEX_OBJECT . ', ' . Database::INDEX_HNSW_EUCLIDEAN . ', ' . Database::INDEX_HNSW_COSINE . ', ' . Database::INDEX_HNSW_DOT . ', '.Database::INDEX_TRIGRAM; return false; @@ -684,7 +684,7 @@ public function checkIdenticalIndexes(Document $index): bool if ($attributesMatch && $ordersMatch) { // Allow fulltext + key/unique combinations (different purposes) - $regularTypes = [Database::INDEX_KEY, Database::INDEX_UNIQUE, Database::INDEX_TTL]; + $regularTypes = [Database::INDEX_KEY, Database::INDEX_UNIQUE]; $isRegularIndex = \in_array($indexType, $regularTypes); $isRegularExisting = \in_array($existingType, $regularTypes); @@ -741,7 +741,8 @@ public function checkObjectIndexes(Document $index): bool return true; } - public function checkTTLIndexes(Document $index): bool{ + public function checkTTLIndexes(Document $index): bool + { $type = $index->getAttribute('type'); $attributes = $index->getAttribute('attributes', []); @@ -765,16 +766,31 @@ public function checkTTLIndexes(Document $index): bool{ $attribute = $this->attributes[\strtolower($attributeName)] ?? new Document(); $attributeType = $attribute->getAttribute('type', ''); - if ($attributeType !== Database::VAR_DATETIME) { - $this->message = 'Object index can only be created on datetime attributes. Attribute "' . $attributeName . '" is of type "' . $attributeType . '"'; + if ($this->supportForAttributes && $attributeType !== Database::VAR_DATETIME) { + $this->message = 'TTL index can only be created on datetime attributes. Attribute "' . $attributeName . '" is of type "' . $attributeType . '"'; return false; } - if($ttl <= 0){ + if ($ttl <= 0) { $this->message = 'TTL must be atleast 1 second'; return false; } + foreach ($this->indexes as $existingIndex) { + $existingAttributes = $existingIndex->getAttribute('attributes', []); + $existingOrders = $existingIndex->getAttribute('orders', []); + $existingType = $existingIndex->getAttribute('type', ''); + if ($this->supportForAttributes && $existingType !== Database::INDEX_TTL) { + continue; + } + $attributeAlreadyPresent = ($this->supportForAttributes && in_array($attribute->getId(), $existingAttributes)) || in_array($attributeName, $existingAttributes); + $ordersMatched = empty(array_diff($existingOrders, $orders)); + if ($attributeAlreadyPresent && $ordersMatched) { + $this->message = 'There is already an index with the same attributes and orders'; + return false; + } + } + return true; } } diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index df31b9595..c3f0055e0 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -7427,4 +7427,265 @@ public function testRegexRedos(): void } $database->deleteCollection($collectionName); } + + public function testTTLIndexes(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if (!$database->getAdapter()->getSupportTTLIndexes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl'); + $database->createCollection($col); + + $database->createAttribute($col, 'expiresAt', Database::VAR_DATETIME, 0, false); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_valid', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 3600 // 1 hour TTL + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(1, $indexes); + $ttlIndex = $indexes[0]; + $this->assertEquals('idx_ttl_valid', $ttlIndex->getId()); + $this->assertEquals(Database::INDEX_TTL, $ttlIndex->getAttribute('type')); + $this->assertEquals(3600, $ttlIndex->getAttribute('ttl')); + + $now = new \DateTime(); + $future1 = (clone $now)->modify('+2 hours'); + $future2 = (clone $now)->modify('+1 hour'); + $past = (clone $now)->modify('-1 hour'); + + $database->createDocuments($col, [ + new Document([ + '$id' => 'doc1', + '$permissions' => $permissions, + 'expiresAt' => $future1->format(\DateTime::ATOM), + ]), + new Document([ + '$id' => 'doc2', + '$permissions' => $permissions, + 'expiresAt' => $future2->format(\DateTime::ATOM), + ]), + new Document([ + '$id' => 'doc3', + '$permissions' => $permissions, + 'expiresAt' => $past->format(\DateTime::ATOM), + ]) + ]); + + $this->assertTrue($database->deleteIndex($col, 'idx_ttl_valid')); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_min', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 1 // Minimum TTL + ) + ); + + $col2 = uniqid('sl_ttl_collection'); + + $expiresAtAttr = new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]); + + $ttlIndexDoc = new Document([ + '$id' => ID::custom('idx_ttl_collection'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200 // 2 hours + ]); + + $database->createCollection($col2, [$expiresAtAttr], [$ttlIndexDoc]); + + $collection2 = $database->getCollection($col2); + $indexes2 = $collection2->getAttribute('indexes'); + $this->assertCount(1, $indexes2); + $ttlIndex2 = $indexes2[0]; + $this->assertEquals('idx_ttl_collection', $ttlIndex2->getId()); + $this->assertEquals(7200, $ttlIndex2->getAttribute('ttl')); + + $database->deleteCollection($col); + $database->deleteCollection($col2); + } + + public function testTTLIndexDuplicatePrevention(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if (!$database->getAdapter()->getSupportTTLIndexes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl_dup'); + $database->createCollection($col); + + $database->createAttribute($col, 'expiresAt', Database::VAR_DATETIME, 0, false); + $database->createAttribute($col, 'deletedAt', Database::VAR_DATETIME, 0, false); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expires', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 3600 // 1 hour + ) + ); + + try { + $database->createIndex( + $col, + 'idx_ttl_expires_duplicate', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 7200 // 2 hours + ); + $this->fail('Expected exception for duplicate TTL index on same attribute'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); + } + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_deleted', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 86400 // 24 hours + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(2, $indexes); + + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); + $this->assertContains('idx_ttl_expires', $indexIds); + $this->assertContains('idx_ttl_deleted', $indexIds); + + try { + $database->createIndex( + $col, + 'idx_ttl_deleted_duplicate', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 172800 // 48 hours + ); + $this->fail('Expected exception for duplicate TTL index on same attribute'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); + } + + $this->assertTrue($database->deleteIndex($col, 'idx_ttl_expires')); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expires_new', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 1800 // 30 minutes + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(2, $indexes); + + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); + $this->assertNotContains('idx_ttl_expires', $indexIds); + $this->assertContains('idx_ttl_expires_new', $indexIds); + $this->assertContains('idx_ttl_deleted', $indexIds); + + $col3 = uniqid('sl_ttl_dup_collection'); + + $expiresAtAttr = new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]); + + $ttlIndex1 = new Document([ + '$id' => ID::custom('idx_ttl_1'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 3600 + ]); + + $ttlIndex2 = new Document([ + '$id' => ID::custom('idx_ttl_2'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200 + ]); + + try { + $database->createCollection($col3, [$expiresAtAttr], [$ttlIndex1, $ttlIndex2]); + $this->fail('Expected exception for duplicate TTL indexes in createCollection'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + // raised in the mongo level + $this->assertStringContainsString('Index already exists', $e->getMessage()); + } + + // Cleanup + $database->deleteCollection($col); + } } diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php index 3c5b0a76a..079a1c7c3 100644 --- a/tests/e2e/Adapter/Scopes/SchemalessTests.php +++ b/tests/e2e/Adapter/Scopes/SchemalessTests.php @@ -1872,7 +1872,6 @@ public function testSchemalessTTLIndexes(): void /** @var Database $database */ $database = static::getDatabase(); - // Only run for MongoDB adapter which supports TTL indexes if ($database->getAdapter()->getSupportForAttributes()) { $this->expectNotToPerformAssertions(); return; @@ -1881,9 +1880,6 @@ public function testSchemalessTTLIndexes(): void $col = uniqid('sl_ttl'); $database->createCollection($col); - // Create datetime attribute for TTL index - $database->createAttribute($col, 'expiresAt', Database::VAR_DATETIME, 0, false); - $permissions = [ Permission::read(Role::any()), Permission::write(Role::any()), @@ -1891,7 +1887,6 @@ public function testSchemalessTTLIndexes(): void Permission::delete(Role::any()) ]; - // Test 1: Create valid TTL index with valid TTL value $this->assertTrue( $database->createIndex( $col, @@ -1904,7 +1899,6 @@ public function testSchemalessTTLIndexes(): void ) ); - // Verify index was created and stored in metadata $collection = $database->getCollection($col); $indexes = $collection->getAttribute('indexes'); $this->assertCount(1, $indexes); @@ -1913,7 +1907,6 @@ public function testSchemalessTTLIndexes(): void $this->assertEquals(Database::INDEX_TTL, $ttlIndex->getAttribute('type')); $this->assertEquals(3600, $ttlIndex->getAttribute('ttl')); - // Test 2: Create documents with expiresAt field $now = new \DateTime(); $future1 = (clone $now)->modify('+2 hours'); $future2 = (clone $now)->modify('+1 hour'); @@ -1945,10 +1938,8 @@ public function testSchemalessTTLIndexes(): void $this->assertEquals('doc2', $doc2->getId()); $this->assertEquals('doc3', $doc3->getId()); - // Test 3: Delete the first TTL index and create a new one with minimum valid TTL (1 second) - // MongoDB only allows one TTL index per attribute, so we need to delete the existing one first $this->assertTrue($database->deleteIndex($col, 'idx_ttl_valid')); - + $this->assertTrue( $database->createIndex( $col, @@ -1961,44 +1952,8 @@ public function testSchemalessTTLIndexes(): void ) ); - // Test 4: Try to create TTL index with TTL = 0 (should fail) - try { - $database->createIndex( - $col, - 'idx_ttl_zero', - Database::INDEX_TTL, - ['expiresAt'], - [], - [Database::ORDER_ASC], - 0 - ); - $this->fail('Expected exception for TTL = 0'); - } catch (Exception $e) { - $this->assertInstanceOf(DatabaseException::class, $e); - $this->assertStringContainsString('TTL must be atleast 1 second', $e->getMessage()); - } - - // Test 5: Try to create TTL index with negative TTL (should fail) - try { - $database->createIndex( - $col, - 'idx_ttl_negative', - Database::INDEX_TTL, - ['expiresAt'], - [], - [Database::ORDER_ASC], - -100 - ); - $this->fail('Expected exception for negative TTL'); - } catch (Exception $e) { - $this->assertInstanceOf(DatabaseException::class, $e); - $this->assertStringContainsString('TTL must be atleast 1 second', $e->getMessage()); - } - - // Test 6: Create TTL index via createCollection with indexes parameter $col2 = uniqid('sl_ttl_collection'); - - // Create attribute document for the collection + $expiresAtAttr = new Document([ '$id' => ID::custom('expiresAt'), 'type' => Database::VAR_DATETIME, @@ -2009,7 +1964,7 @@ public function testSchemalessTTLIndexes(): void 'array' => false, 'filters' => ['datetime'], ]); - + $ttlIndexDoc = new Document([ '$id' => ID::custom('idx_ttl_collection'), 'type' => Database::INDEX_TTL, @@ -2021,7 +1976,6 @@ public function testSchemalessTTLIndexes(): void $database->createCollection($col2, [$expiresAtAttr], [$ttlIndexDoc]); - // Verify TTL index was created via createCollection $collection2 = $database->getCollection($col2); $indexes2 = $collection2->getAttribute('indexes'); $this->assertCount(1, $indexes2); @@ -2029,7 +1983,6 @@ public function testSchemalessTTLIndexes(): void $this->assertEquals('idx_ttl_collection', $ttlIndex2->getId()); $this->assertEquals(7200, $ttlIndex2->getAttribute('ttl')); - // Cleanup $database->deleteCollection($col); $database->deleteCollection($col2); } @@ -2039,7 +1992,6 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void /** @var Database $database */ $database = static::getDatabase(); - // Only run for MongoDB adapter which supports TTL indexes if ($database->getAdapter()->getSupportForAttributes()) { $this->expectNotToPerformAssertions(); return; @@ -2048,11 +2000,6 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void $col = uniqid('sl_ttl_dup'); $database->createCollection($col); - // Create datetime attribute for TTL index - $database->createAttribute($col, 'expiresAt', Database::VAR_DATETIME, 0, false); - $database->createAttribute($col, 'deletedAt', Database::VAR_DATETIME, 0, false); - - // Test 1: Create first TTL index on expiresAt $this->assertTrue( $database->createIndex( $col, @@ -2065,7 +2012,6 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void ) ); - // Test 2: Try to create another TTL index on the same attribute (should fail) try { $database->createIndex( $col, @@ -2079,10 +2025,9 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void $this->fail('Expected exception for duplicate TTL index on same attribute'); } catch (Exception $e) { $this->assertInstanceOf(DatabaseException::class, $e); - $this->assertStringContainsString('A TTL index already exists on attribute', $e->getMessage()); + $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); } - // Test 3: Create TTL index on different attribute (should succeed) $this->assertTrue( $database->createIndex( $col, @@ -2095,16 +2040,14 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void ) ); - // Verify both indexes exist $collection = $database->getCollection($col); $indexes = $collection->getAttribute('indexes'); $this->assertCount(2, $indexes); - + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); $this->assertContains('idx_ttl_expires', $indexIds); $this->assertContains('idx_ttl_deleted', $indexIds); - // Test 4: Try to create another TTL index on deletedAt (should fail) try { $database->createIndex( $col, @@ -2118,12 +2061,11 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void $this->fail('Expected exception for duplicate TTL index on same attribute'); } catch (Exception $e) { $this->assertInstanceOf(DatabaseException::class, $e); - $this->assertStringContainsString('A TTL index already exists on attribute', $e->getMessage()); + $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); } - // Test 5: Delete first TTL index and create a new one on same attribute (should succeed) $this->assertTrue($database->deleteIndex($col, 'idx_ttl_expires')); - + $this->assertTrue( $database->createIndex( $col, @@ -2136,19 +2078,17 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void ) ); - // Verify the new index replaced the old one $collection = $database->getCollection($col); $indexes = $collection->getAttribute('indexes'); $this->assertCount(2, $indexes); - + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); $this->assertNotContains('idx_ttl_expires', $indexIds); $this->assertContains('idx_ttl_expires_new', $indexIds); $this->assertContains('idx_ttl_deleted', $indexIds); - // Test 6: Try to create TTL index via createCollection with duplicate (should fail validation) $col3 = uniqid('sl_ttl_dup_collection'); - + $expiresAtAttr = new Document([ '$id' => ID::custom('expiresAt'), 'type' => Database::VAR_DATETIME, @@ -2159,7 +2099,7 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void 'array' => false, 'filters' => ['datetime'], ]); - + $ttlIndex1 = new Document([ '$id' => ID::custom('idx_ttl_1'), 'type' => Database::INDEX_TTL, @@ -2183,10 +2123,9 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void $this->fail('Expected exception for duplicate TTL indexes in createCollection'); } catch (Exception $e) { $this->assertInstanceOf(DatabaseException::class, $e); - $this->assertStringContainsString('A TTL index already exists on attribute', $e->getMessage()); + $this->assertStringContainsString('Index already exists', $e->getMessage()); } - // Cleanup $database->deleteCollection($col); } } diff --git a/tests/unit/Validator/IndexTest.php b/tests/unit/Validator/IndexTest.php index b8bfee056..6ade7a362 100644 --- a/tests/unit/Validator/IndexTest.php +++ b/tests/unit/Validator/IndexTest.php @@ -758,7 +758,7 @@ public function testTTLIndexValidation(): void true, // supportForFulltextIndexes true // supportForTTLIndexes ); - + $duplicateTTLIndex = new Document([ '$id' => ID::custom('idx_ttl_duplicate'), 'type' => Database::INDEX_TTL, @@ -768,7 +768,7 @@ public function testTTLIndexValidation(): void 'ttl' => 7200, ]); $this->assertFalse($validatorWithExisting->isValid($duplicateTTLIndex)); - $this->assertStringContainsString('A TTL index already exists on attribute', $validatorWithExisting->getDescription()); + $this->assertStringContainsString('There is already an index with the same attributes and orders', $validatorWithExisting->getDescription()); // Validator with supportForTrigramIndexes disabled should reject TTL $validatorNoSupport = new Index($collection->getAttribute('attributes'), $collection->getAttribute('indexes', []), 768, [], false, false, false, false, false, false, false, false, false); From 151703fc29f8903c5516b543987455418a8b9a1a Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 12 Jan 2026 13:41:06 +0530 Subject: [PATCH 03/16] updated adapters with the support method for the ttl index support --- src/Database/Adapter/MariaDB.php | 5 +++++ src/Database/Adapter/MySQL.php | 5 +++++ src/Database/Adapter/Pool.php | 5 +++++ src/Database/Adapter/Postgres.php | 5 +++++ src/Database/Adapter/SQLite.php | 5 +++++ 5 files changed, 25 insertions(+) diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index ef44171c6..242b0d9ad 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -2255,4 +2255,9 @@ public function getSupportForPOSIXRegex(): bool { return false; } + + public function getSupportTTLIndexes(): bool + { + return false; + } } diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index 308013738..d5740cf66 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -317,4 +317,9 @@ protected function getOperatorSQL(string $column, \Utopia\Database\Operator $ope // For all other operators, use parent implementation return parent::getOperatorSQL($column, $operator, $bindIndex); } + + public function getSupportTTLIndexes(): bool + { + return false; + } } diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index e6bf77681..1d8c892e6 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -649,4 +649,9 @@ public function getSupportForAlterLocks(): bool { return $this->delegate(__FUNCTION__, \func_get_args()); } + + public function getSupportTTLIndexes(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 38c3c3a26..5cfe39d55 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -2832,4 +2832,9 @@ protected function getSQLTable(string $name): string return "{$this->quote($this->getDatabase())}.{$this->quote($table)}"; } + + public function getSupportTTLIndexes(): bool + { + return false; + } } diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 538b93a68..55e195dc2 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -1909,4 +1909,9 @@ public function getSupportForPOSIXRegex(): bool { return false; } + + public function getSupportTTLIndexes(): bool + { + return false; + } } From cc63ade737a8f66599af0e594a4ddba1aefcca6f Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 12 Jan 2026 14:09:12 +0530 Subject: [PATCH 04/16] fix: update TTL index validation messages and conditions for shared tables --- src/Database/Adapter/Mongo.php | 4 +- src/Database/Validator/Index.php | 2 +- tests/e2e/Adapter/Scopes/DocumentTests.php | 261 --------------------- tests/e2e/Adapter/Scopes/IndexTests.php | 261 +++++++++++++++++++++ tests/unit/Validator/IndexTest.php | 4 +- 5 files changed, 266 insertions(+), 266 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index cf3d28189..68935742d 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -488,7 +488,7 @@ public function createCollection(string $name, array $attributes = [], array $in $orders = $index->getAttribute('orders'); // If sharedTables, always add _tenant as the first key - if ($this->sharedTables) { + if ($this->sharedTables && $index->getAttribute('type') !== Database::INDEX_TTL) { $key['_tenant'] = $this->getOrder(Database::ORDER_ASC); } @@ -925,7 +925,7 @@ public function createIndex(string $collection, string $id, string $type, array $indexes['name'] = $id; // If sharedTables, always add _tenant as the first key - if ($this->sharedTables) { + if ($this->sharedTables && $type !== Database::INDEX_TTL) { $indexes['key']['_tenant'] = $this->getOrder(Database::ORDER_ASC); } diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php index 5226d3300..35d0f78d8 100644 --- a/src/Database/Validator/Index.php +++ b/src/Database/Validator/Index.php @@ -753,7 +753,7 @@ public function checkTTLIndexes(Document $index): bool } if (count($attributes) !== 1) { - $this->message = 'TTL index can be created on a single object attribute'; + $this->message = 'TTL index can be created on a single datetime attribute'; return false; } diff --git a/tests/e2e/Adapter/Scopes/DocumentTests.php b/tests/e2e/Adapter/Scopes/DocumentTests.php index c3f0055e0..df31b9595 100644 --- a/tests/e2e/Adapter/Scopes/DocumentTests.php +++ b/tests/e2e/Adapter/Scopes/DocumentTests.php @@ -7427,265 +7427,4 @@ public function testRegexRedos(): void } $database->deleteCollection($collectionName); } - - public function testTTLIndexes(): void - { - /** @var Database $database */ - $database = static::getDatabase(); - - if (!$database->getAdapter()->getSupportTTLIndexes()) { - $this->expectNotToPerformAssertions(); - return; - } - - $col = uniqid('sl_ttl'); - $database->createCollection($col); - - $database->createAttribute($col, 'expiresAt', Database::VAR_DATETIME, 0, false); - - $permissions = [ - Permission::read(Role::any()), - Permission::write(Role::any()), - Permission::update(Role::any()), - Permission::delete(Role::any()) - ]; - - $this->assertTrue( - $database->createIndex( - $col, - 'idx_ttl_valid', - Database::INDEX_TTL, - ['expiresAt'], - [], - [Database::ORDER_ASC], - 3600 // 1 hour TTL - ) - ); - - $collection = $database->getCollection($col); - $indexes = $collection->getAttribute('indexes'); - $this->assertCount(1, $indexes); - $ttlIndex = $indexes[0]; - $this->assertEquals('idx_ttl_valid', $ttlIndex->getId()); - $this->assertEquals(Database::INDEX_TTL, $ttlIndex->getAttribute('type')); - $this->assertEquals(3600, $ttlIndex->getAttribute('ttl')); - - $now = new \DateTime(); - $future1 = (clone $now)->modify('+2 hours'); - $future2 = (clone $now)->modify('+1 hour'); - $past = (clone $now)->modify('-1 hour'); - - $database->createDocuments($col, [ - new Document([ - '$id' => 'doc1', - '$permissions' => $permissions, - 'expiresAt' => $future1->format(\DateTime::ATOM), - ]), - new Document([ - '$id' => 'doc2', - '$permissions' => $permissions, - 'expiresAt' => $future2->format(\DateTime::ATOM), - ]), - new Document([ - '$id' => 'doc3', - '$permissions' => $permissions, - 'expiresAt' => $past->format(\DateTime::ATOM), - ]) - ]); - - $this->assertTrue($database->deleteIndex($col, 'idx_ttl_valid')); - - $this->assertTrue( - $database->createIndex( - $col, - 'idx_ttl_min', - Database::INDEX_TTL, - ['expiresAt'], - [], - [Database::ORDER_ASC], - 1 // Minimum TTL - ) - ); - - $col2 = uniqid('sl_ttl_collection'); - - $expiresAtAttr = new Document([ - '$id' => ID::custom('expiresAt'), - 'type' => Database::VAR_DATETIME, - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ]); - - $ttlIndexDoc = new Document([ - '$id' => ID::custom('idx_ttl_collection'), - 'type' => Database::INDEX_TTL, - 'attributes' => ['expiresAt'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - 'ttl' => 7200 // 2 hours - ]); - - $database->createCollection($col2, [$expiresAtAttr], [$ttlIndexDoc]); - - $collection2 = $database->getCollection($col2); - $indexes2 = $collection2->getAttribute('indexes'); - $this->assertCount(1, $indexes2); - $ttlIndex2 = $indexes2[0]; - $this->assertEquals('idx_ttl_collection', $ttlIndex2->getId()); - $this->assertEquals(7200, $ttlIndex2->getAttribute('ttl')); - - $database->deleteCollection($col); - $database->deleteCollection($col2); - } - - public function testTTLIndexDuplicatePrevention(): void - { - /** @var Database $database */ - $database = static::getDatabase(); - - if (!$database->getAdapter()->getSupportTTLIndexes()) { - $this->expectNotToPerformAssertions(); - return; - } - - $col = uniqid('sl_ttl_dup'); - $database->createCollection($col); - - $database->createAttribute($col, 'expiresAt', Database::VAR_DATETIME, 0, false); - $database->createAttribute($col, 'deletedAt', Database::VAR_DATETIME, 0, false); - - $this->assertTrue( - $database->createIndex( - $col, - 'idx_ttl_expires', - Database::INDEX_TTL, - ['expiresAt'], - [], - [Database::ORDER_ASC], - 3600 // 1 hour - ) - ); - - try { - $database->createIndex( - $col, - 'idx_ttl_expires_duplicate', - Database::INDEX_TTL, - ['expiresAt'], - [], - [Database::ORDER_ASC], - 7200 // 2 hours - ); - $this->fail('Expected exception for duplicate TTL index on same attribute'); - } catch (Exception $e) { - $this->assertInstanceOf(DatabaseException::class, $e); - $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); - } - - $this->assertTrue( - $database->createIndex( - $col, - 'idx_ttl_deleted', - Database::INDEX_TTL, - ['deletedAt'], - [], - [Database::ORDER_ASC], - 86400 // 24 hours - ) - ); - - $collection = $database->getCollection($col); - $indexes = $collection->getAttribute('indexes'); - $this->assertCount(2, $indexes); - - $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); - $this->assertContains('idx_ttl_expires', $indexIds); - $this->assertContains('idx_ttl_deleted', $indexIds); - - try { - $database->createIndex( - $col, - 'idx_ttl_deleted_duplicate', - Database::INDEX_TTL, - ['deletedAt'], - [], - [Database::ORDER_ASC], - 172800 // 48 hours - ); - $this->fail('Expected exception for duplicate TTL index on same attribute'); - } catch (Exception $e) { - $this->assertInstanceOf(DatabaseException::class, $e); - $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); - } - - $this->assertTrue($database->deleteIndex($col, 'idx_ttl_expires')); - - $this->assertTrue( - $database->createIndex( - $col, - 'idx_ttl_expires_new', - Database::INDEX_TTL, - ['expiresAt'], - [], - [Database::ORDER_ASC], - 1800 // 30 minutes - ) - ); - - $collection = $database->getCollection($col); - $indexes = $collection->getAttribute('indexes'); - $this->assertCount(2, $indexes); - - $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); - $this->assertNotContains('idx_ttl_expires', $indexIds); - $this->assertContains('idx_ttl_expires_new', $indexIds); - $this->assertContains('idx_ttl_deleted', $indexIds); - - $col3 = uniqid('sl_ttl_dup_collection'); - - $expiresAtAttr = new Document([ - '$id' => ID::custom('expiresAt'), - 'type' => Database::VAR_DATETIME, - 'size' => 0, - 'signed' => false, - 'required' => false, - 'default' => null, - 'array' => false, - 'filters' => ['datetime'], - ]); - - $ttlIndex1 = new Document([ - '$id' => ID::custom('idx_ttl_1'), - 'type' => Database::INDEX_TTL, - 'attributes' => ['expiresAt'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - 'ttl' => 3600 - ]); - - $ttlIndex2 = new Document([ - '$id' => ID::custom('idx_ttl_2'), - 'type' => Database::INDEX_TTL, - 'attributes' => ['expiresAt'], - 'lengths' => [], - 'orders' => [Database::ORDER_ASC], - 'ttl' => 7200 - ]); - - try { - $database->createCollection($col3, [$expiresAtAttr], [$ttlIndex1, $ttlIndex2]); - $this->fail('Expected exception for duplicate TTL indexes in createCollection'); - } catch (Exception $e) { - $this->assertInstanceOf(DatabaseException::class, $e); - // raised in the mongo level - $this->assertStringContainsString('Index already exists', $e->getMessage()); - } - - // Cleanup - $database->deleteCollection($col); - } } diff --git a/tests/e2e/Adapter/Scopes/IndexTests.php b/tests/e2e/Adapter/Scopes/IndexTests.php index e5eda16d0..6c5e21aaa 100644 --- a/tests/e2e/Adapter/Scopes/IndexTests.php +++ b/tests/e2e/Adapter/Scopes/IndexTests.php @@ -784,4 +784,265 @@ public function testTrigramIndexValidation(): void $database->deleteCollection($collectionId); } } + + public function testTTLIndexes(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if (!$database->getAdapter()->getSupportTTLIndexes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl'); + $database->createCollection($col); + + $database->createAttribute($col, 'expiresAt', Database::VAR_DATETIME, 0, false); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_valid', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 3600 // 1 hour TTL + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(1, $indexes); + $ttlIndex = $indexes[0]; + $this->assertEquals('idx_ttl_valid', $ttlIndex->getId()); + $this->assertEquals(Database::INDEX_TTL, $ttlIndex->getAttribute('type')); + $this->assertEquals(3600, $ttlIndex->getAttribute('ttl')); + + $now = new \DateTime(); + $future1 = (clone $now)->modify('+2 hours'); + $future2 = (clone $now)->modify('+1 hour'); + $past = (clone $now)->modify('-1 hour'); + + $database->createDocuments($col, [ + new Document([ + '$id' => 'doc1', + '$permissions' => $permissions, + 'expiresAt' => $future1->format(\DateTime::ATOM), + ]), + new Document([ + '$id' => 'doc2', + '$permissions' => $permissions, + 'expiresAt' => $future2->format(\DateTime::ATOM), + ]), + new Document([ + '$id' => 'doc3', + '$permissions' => $permissions, + 'expiresAt' => $past->format(\DateTime::ATOM), + ]) + ]); + + $this->assertTrue($database->deleteIndex($col, 'idx_ttl_valid')); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_min', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 1 // Minimum TTL + ) + ); + + $col2 = uniqid('sl_ttl_collection'); + + $expiresAtAttr = new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]); + + $ttlIndexDoc = new Document([ + '$id' => ID::custom('idx_ttl_collection'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200 // 2 hours + ]); + + $database->createCollection($col2, [$expiresAtAttr], [$ttlIndexDoc]); + + $collection2 = $database->getCollection($col2); + $indexes2 = $collection2->getAttribute('indexes'); + $this->assertCount(1, $indexes2); + $ttlIndex2 = $indexes2[0]; + $this->assertEquals('idx_ttl_collection', $ttlIndex2->getId()); + $this->assertEquals(7200, $ttlIndex2->getAttribute('ttl')); + + $database->deleteCollection($col); + $database->deleteCollection($col2); + } + + public function testTTLIndexDuplicatePrevention(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if (!$database->getAdapter()->getSupportTTLIndexes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl_dup'); + $database->createCollection($col); + + $database->createAttribute($col, 'expiresAt', Database::VAR_DATETIME, 0, false); + $database->createAttribute($col, 'deletedAt', Database::VAR_DATETIME, 0, false); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expires', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 3600 // 1 hour + ) + ); + + try { + $database->createIndex( + $col, + 'idx_ttl_expires_duplicate', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 7200 // 2 hours + ); + $this->fail('Expected exception for duplicate TTL index on same attribute'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); + } + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_deleted', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 86400 // 24 hours + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(2, $indexes); + + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); + $this->assertContains('idx_ttl_expires', $indexIds); + $this->assertContains('idx_ttl_deleted', $indexIds); + + try { + $database->createIndex( + $col, + 'idx_ttl_deleted_duplicate', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 172800 // 48 hours + ); + $this->fail('Expected exception for duplicate TTL index on same attribute'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); + } + + $this->assertTrue($database->deleteIndex($col, 'idx_ttl_expires')); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expires_new', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 1800 // 30 minutes + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(2, $indexes); + + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); + $this->assertNotContains('idx_ttl_expires', $indexIds); + $this->assertContains('idx_ttl_expires_new', $indexIds); + $this->assertContains('idx_ttl_deleted', $indexIds); + + $col3 = uniqid('sl_ttl_dup_collection'); + + $expiresAtAttr = new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]); + + $ttlIndex1 = new Document([ + '$id' => ID::custom('idx_ttl_1'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 3600 + ]); + + $ttlIndex2 = new Document([ + '$id' => ID::custom('idx_ttl_2'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200 + ]); + + try { + $database->createCollection($col3, [$expiresAtAttr], [$ttlIndex1, $ttlIndex2]); + $this->fail('Expected exception for duplicate TTL indexes in createCollection'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + // raised in the mongo level + $this->assertStringContainsString('Index already exists', $e->getMessage()); + } + + // Cleanup + $database->deleteCollection($col); + } } diff --git a/tests/unit/Validator/IndexTest.php b/tests/unit/Validator/IndexTest.php index 6ade7a362..e6aeb3c56 100644 --- a/tests/unit/Validator/IndexTest.php +++ b/tests/unit/Validator/IndexTest.php @@ -699,7 +699,7 @@ public function testTTLIndexValidation(): void 'ttl' => 3600, ]); $this->assertFalse($validator->isValid($invalidIndexType)); - $this->assertStringContainsString('Object index can only be created on datetime attributes', $validator->getDescription()); + $this->assertStringContainsString('TTL index can only be created on datetime attributes', $validator->getDescription()); // Invalid: TTL index on multiple attributes $invalidIndexMulti = new Document([ @@ -711,7 +711,7 @@ public function testTTLIndexValidation(): void 'ttl' => 3600, ]); $this->assertFalse($validator->isValid($invalidIndexMulti)); - $this->assertStringContainsString('TTL index can be created on a single object attribute', $validator->getDescription()); + $this->assertStringContainsString('TTL index can be created on a single datetime attribute', $validator->getDescription()); // Invalid: TTL index without orders $invalidIndexNoOrders = new Document([ From ed322e6b89e2dc0808ca392dc513ca24d1b52198 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 12 Jan 2026 14:23:15 +0530 Subject: [PATCH 05/16] typo --- src/Database/Validator/Index.php | 2 +- tests/unit/Validator/IndexTest.php | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php index 35d0f78d8..ed7ffea74 100644 --- a/src/Database/Validator/Index.php +++ b/src/Database/Validator/Index.php @@ -772,7 +772,7 @@ public function checkTTLIndexes(Document $index): bool } if ($ttl <= 0) { - $this->message = 'TTL must be atleast 1 second'; + $this->message = 'TTL must be at least 1 second'; return false; } diff --git a/tests/unit/Validator/IndexTest.php b/tests/unit/Validator/IndexTest.php index e6aeb3c56..664874051 100644 --- a/tests/unit/Validator/IndexTest.php +++ b/tests/unit/Validator/IndexTest.php @@ -675,7 +675,7 @@ public function testTTLIndexValidation(): void 'ttl' => 0, ]); $this->assertFalse($validator->isValid($invalidIndexZero)); - $this->assertEquals('TTL must be atleast 1 second', $validator->getDescription()); + $this->assertEquals('TTL must be at least 1 second', $validator->getDescription()); // Invalid: TTL index with TTL < 0 $invalidIndexNegative = new Document([ @@ -687,7 +687,7 @@ public function testTTLIndexValidation(): void 'ttl' => -100, ]); $this->assertFalse($validator->isValid($invalidIndexNegative)); - $this->assertEquals('TTL must be atleast 1 second', $validator->getDescription()); + $this->assertEquals('TTL must be at least 1 second', $validator->getDescription()); // Invalid: TTL index on non-datetime attribute $invalidIndexType = new Document([ From 8fa2454e25d20cee1ecd74468f7828fa09bfabfe Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Mon, 12 Jan 2026 14:28:56 +0530 Subject: [PATCH 06/16] added ttl index in the missing index message --- src/Database/Validator/Index.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php index ed7ffea74..1e367d1ad 100644 --- a/src/Database/Validator/Index.php +++ b/src/Database/Validator/Index.php @@ -234,7 +234,7 @@ public function checkValidIndex(Document $index): bool break; default: - $this->message = 'Unknown index type: ' . $type . '. Must be one of ' . Database::INDEX_KEY . ', ' . Database::INDEX_UNIQUE . ', ' . Database::INDEX_FULLTEXT . ', ' . Database::INDEX_SPATIAL . ', ' . Database::INDEX_OBJECT . ', ' . Database::INDEX_HNSW_EUCLIDEAN . ', ' . Database::INDEX_HNSW_COSINE . ', ' . Database::INDEX_HNSW_DOT . ', '.Database::INDEX_TRIGRAM; + $this->message = 'Unknown index type: ' . $type . '. Must be one of ' . Database::INDEX_KEY . ', ' . Database::INDEX_UNIQUE . ', ' . Database::INDEX_FULLTEXT . ', ' . Database::INDEX_SPATIAL . ', ' . Database::INDEX_OBJECT . ', ' . Database::INDEX_HNSW_EUCLIDEAN . ', ' . Database::INDEX_HNSW_COSINE . ', ' . Database::INDEX_HNSW_DOT . ', '.Database::INDEX_TRIGRAM . ', '.Database::INDEX_TTL; return false; } return true; From 07ccad8d44d3ac21e116ecf64eaa96e4685a5075 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 13 Jan 2026 20:53:16 +0530 Subject: [PATCH 07/16] Refactor SchemalessTests to improve datetime handling and add new tests - Updated assertions in existing tests to compare datetime values by parsing them into DateTime objects, ensuring accurate comparisons with MongoDB's format. - Added new test `testSchemalessDatetimeCreationAndFetching` to verify creation, fetching, and updating of ISO 8601 datetime strings. - Introduced `testSchemalessTTLExpiry` to validate TTL functionality with mixed document types, ensuring expired documents are correctly removed. - Implemented `testStringAndDatetime` and `testStringAndDateWithTTL` to check behavior of string and datetime fields, including handling of TTL indexes. --- src/Database/Adapter/Mongo.php | 163 ++++- tests/e2e/Adapter/Scopes/SchemalessTests.php | 685 ++++++++++++++++++- 2 files changed, 791 insertions(+), 57 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 68935742d..ecb050f8e 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -1254,30 +1254,7 @@ public function castingAfter(Document $collection, Document $document): Document $node = (int)$node; break; case Database::VAR_DATETIME: - if ($node instanceof UTCDateTime) { - // Handle UTCDateTime objects - $node = DateTime::format($node->toDateTime()); - } elseif (is_array($node) && isset($node['$date'])) { - // Handle Extended JSON format from (array) cast - // Format: {"$date":{"$numberLong":"1760405478290"}} - if (is_array($node['$date']) && isset($node['$date']['$numberLong'])) { - $milliseconds = (int)$node['$date']['$numberLong']; - $seconds = intdiv($milliseconds, 1000); - $microseconds = ($milliseconds % 1000) * 1000; - $dateTime = \DateTime::createFromFormat('U.u', $seconds . '.' . str_pad((string)$microseconds, 6, '0')); - if ($dateTime) { - $dateTime->setTimezone(new \DateTimeZone('UTC')); - $node = DateTime::format($dateTime); - } - } - } elseif (is_string($node)) { - // Already a string, validate and pass through - try { - new \DateTime($node); - } catch (\Exception $e) { - // Invalid date string, skip - } - } + $node = $this->convertUTCDateToString($node); break; case Database::VAR_OBJECT: // Convert stdClass objects to arrays for object attributes @@ -1298,6 +1275,8 @@ public function castingAfter(Document $collection, Document $document): Document // mongodb results out a stdclass for objects if (is_object($value) && get_class($value) === stdClass::class) { $document->setAttribute($key, $this->convertStdClassToArray($value)); + } elseif ($value instanceof UTCDateTime) { + $document->setAttribute($key, $this->convertUTCDateToString($value)); } } } @@ -1380,6 +1359,24 @@ public function castingBefore(Document $collection, Document $document): Documen unset($node); $document->setAttribute($key, ($array) ? $value : $value[0]); } + $indexes = $collection->getAttribute('indexes'); + $ttlIndexes = array_filter($indexes, fn ($index) => $index->getAttribute('type') === Database::INDEX_TTL); + + if (!$this->getSupportForAttributes()) { + foreach ($document->getArrayCopy() as $key => $value) { + if (in_array($this->getInternalKeyForAttribute($key), Database::INTERNAL_ATTRIBUTE_KEYS)) { + continue; + } + if (is_string($value) && (in_array($key, $ttlIndexes) || $this->isExtendedISODatetime($value))) { + try { + $newValue = new UTCDateTime(new \DateTime($value)); + $document->setAttribute($key, $newValue); + } catch (\Throwable $th) { + // skip -> a valid string + } + } + } + } return $document; } @@ -3419,4 +3416,122 @@ public function getSupportTTLIndexes(): bool { return true; } + + protected function isExtendedISODatetime(string $val): bool + { + /** + * Min: + * YYYY-MM-DDTHH:mm:ssZ (20) + * YYYY-MM-DDTHH:mm:ss+HH:MM (25) + * + * Max: + * YYYY-MM-DDTHH:mm:ss.fffffZ (26) + * YYYY-MM-DDTHH:mm:ss.fffff+HH:MM (31) + */ + + $len = strlen($val); + + // absolute minimum + if ($len < 20) { + return false; + } + + // fixed datetime fingerprints + if ( + !isset($val[19]) || + $val[4] !== '-' || + $val[7] !== '-' || + $val[10] !== 'T' || + $val[13] !== ':' || + $val[16] !== ':' + ) { + return false; + } + + // timezone detection + $hasZ = ($val[$len - 1] === 'Z'); + + $hasOffset = ( + $len >= 25 && + ($val[$len - 6] === '+' || $val[$len - 6] === '-') && + $val[$len - 3] === ':' + ); + + if (!$hasZ && !$hasOffset) { + return false; + } + + if ($hasZ && $len > 26) { + return false; + } + if ($hasOffset && ($len < 25 || $len > 31)) { + return false; + } + + $digitPositions = [ + 0,1,2,3, + 5,6, + 8,9, + 11,12, + 14,15, + 17,18 + ]; + + $timeEnd = $hasZ ? $len - 1 : $len - 6; + + // fractional seconds + if ($timeEnd > 19) { + if ($val[19] !== '.' || $timeEnd < 21) { + return false; + } + for ($i = 20; $i < $timeEnd; $i++) { + $digitPositions[] = $i; + } + } + + // timezone offset numeric digits + if ($hasOffset) { + foreach ([$len - 5, $len - 4, $len - 2, $len - 1] as $i) { + $digitPositions[] = $i; + } + } + + foreach ($digitPositions as $i) { + if (!ctype_digit($val[$i])) { + return false; + } + } + + return true; + } + + protected function convertUTCDateToString(mixed $node): mixed + { + if ($node instanceof UTCDateTime) { + // Handle UTCDateTime objects + $node = DateTime::format($node->toDateTime()); + } elseif (is_array($node) && isset($node['$date'])) { + // Handle Extended JSON format from (array) cast + // Format: {"$date":{"$numberLong":"1760405478290"}} + if (is_array($node['$date']) && isset($node['$date']['$numberLong'])) { + $milliseconds = (int)$node['$date']['$numberLong']; + $seconds = intdiv($milliseconds, 1000); + $microseconds = ($milliseconds % 1000) * 1000; + $dateTime = \DateTime::createFromFormat('U.u', $seconds . '.' . str_pad((string)$microseconds, 6, '0')); + if ($dateTime) { + $dateTime->setTimezone(new \DateTimeZone('UTC')); + $node = DateTime::format($dateTime); + } + } + } elseif (is_string($node)) { + // Already a string, validate and pass through + try { + new \DateTime($node); + } catch (\Exception $e) { + // Invalid date string, skip + } + } + + return $node; + } } diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php index 079a1c7c3..9bac0d8fe 100644 --- a/tests/e2e/Adapter/Scopes/SchemalessTests.php +++ b/tests/e2e/Adapter/Scopes/SchemalessTests.php @@ -1030,19 +1030,36 @@ public function testSchemalessDates(): void $this->assertEquals('d1', $doc1->getId()); $this->assertTrue(is_string($doc1->getAttribute('curDate'))); - $this->assertEquals($curDate1, $doc1->getAttribute('curDate')); + // MongoDB converts ISO 8601 to 'Y-m-d H:i:s.v' format, so compare by parsing + $curDate1Value = $doc1->getAttribute('curDate'); + $parsedCurDate1 = new \DateTime($curDate1Value); + $parsedExpectedCurDate1 = new \DateTime($curDate1); + $this->assertEquals($parsedExpectedCurDate1->getTimestamp(), $parsedCurDate1->getTimestamp()); $this->assertTrue(is_string($doc1->getAttribute('$createdAt'))); $this->assertTrue(is_string($doc1->getAttribute('$updatedAt'))); - $this->assertEquals($createdAt1, $doc1->getAttribute('$createdAt')); - $this->assertEquals($updatedAt1, $doc1->getAttribute('$updatedAt')); + // Internal attributes should preserve format better, but verify by parsing for MongoDB + $createdAt1Value = $doc1->getAttribute('$createdAt'); + $updatedAt1Value = $doc1->getAttribute('$updatedAt'); + $parsedCreatedAt1 = new \DateTime($createdAt1Value); + $parsedUpdatedAt1 = new \DateTime($updatedAt1Value); + $parsedExpectedCreatedAt1 = new \DateTime($createdAt1); + $parsedExpectedUpdatedAt1 = new \DateTime($updatedAt1); + $this->assertEquals($parsedExpectedCreatedAt1->getTimestamp(), $parsedCreatedAt1->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt1->getTimestamp(), $parsedUpdatedAt1->getTimestamp()); $fetched1 = $database->getDocument($col, 'd1'); - $this->assertEquals($curDate1, $fetched1->getAttribute('curDate')); - $this->assertTrue(is_string($fetched1->getAttribute('curDate'))); + $fetchedCurDate1 = $fetched1->getAttribute('curDate'); + $this->assertTrue(is_string($fetchedCurDate1)); + $parsedFetchedCurDate1 = new \DateTime($fetchedCurDate1); + $this->assertEquals($parsedExpectedCurDate1->getTimestamp(), $parsedFetchedCurDate1->getTimestamp()); $this->assertTrue(is_string($fetched1->getAttribute('$createdAt'))); $this->assertTrue(is_string($fetched1->getAttribute('$updatedAt'))); - $this->assertEquals($createdAt1, $fetched1->getAttribute('$createdAt')); - $this->assertEquals($updatedAt1, $fetched1->getAttribute('$updatedAt')); + $fetchedCreatedAt1 = $fetched1->getAttribute('$createdAt'); + $fetchedUpdatedAt1 = $fetched1->getAttribute('$updatedAt'); + $parsedFetchedCreatedAt1 = new \DateTime($fetchedCreatedAt1); + $parsedFetchedUpdatedAt1 = new \DateTime($fetchedUpdatedAt1); + $this->assertEquals($parsedExpectedCreatedAt1->getTimestamp(), $parsedFetchedCreatedAt1->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt1->getTimestamp(), $parsedFetchedUpdatedAt1->getTimestamp()); // createDocuments with preserved dates $createdAt2 = '2001-02-03T04:05:06.000+00:00'; @@ -1074,14 +1091,32 @@ public function testSchemalessDates(): void $this->assertEquals(2, $countCreated); $fetched2 = $database->getDocument($col, 'd2'); - $this->assertEquals($curDate2, $fetched2->getAttribute('curDate')); - $this->assertEquals($createdAt2, $fetched2->getAttribute('$createdAt')); - $this->assertEquals($updatedAt2, $fetched2->getAttribute('$updatedAt')); + $fetchedCurDate2 = $fetched2->getAttribute('curDate'); + $parsedCurDate2 = new \DateTime($fetchedCurDate2); + $parsedExpectedCurDate2 = new \DateTime($curDate2); + $this->assertEquals($parsedExpectedCurDate2->getTimestamp(), $parsedCurDate2->getTimestamp()); + $fetchedCreatedAt2 = $fetched2->getAttribute('$createdAt'); + $fetchedUpdatedAt2 = $fetched2->getAttribute('$updatedAt'); + $parsedCreatedAt2 = new \DateTime($fetchedCreatedAt2); + $parsedUpdatedAt2 = new \DateTime($fetchedUpdatedAt2); + $parsedExpectedCreatedAt2 = new \DateTime($createdAt2); + $parsedExpectedUpdatedAt2 = new \DateTime($updatedAt2); + $this->assertEquals($parsedExpectedCreatedAt2->getTimestamp(), $parsedCreatedAt2->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt2->getTimestamp(), $parsedUpdatedAt2->getTimestamp()); $fetched3 = $database->getDocument($col, 'd3'); - $this->assertEquals($curDate3, $fetched3->getAttribute('curDate')); - $this->assertEquals($createdAt3, $fetched3->getAttribute('$createdAt')); - $this->assertEquals($updatedAt3, $fetched3->getAttribute('$updatedAt')); + $fetchedCurDate3 = $fetched3->getAttribute('curDate'); + $parsedCurDate3 = new \DateTime($fetchedCurDate3); + $parsedExpectedCurDate3 = new \DateTime($curDate3); + $this->assertEquals($parsedExpectedCurDate3->getTimestamp(), $parsedCurDate3->getTimestamp()); + $fetchedCreatedAt3 = $fetched3->getAttribute('$createdAt'); + $fetchedUpdatedAt3 = $fetched3->getAttribute('$updatedAt'); + $parsedCreatedAt3 = new \DateTime($fetchedCreatedAt3); + $parsedUpdatedAt3 = new \DateTime($fetchedUpdatedAt3); + $parsedExpectedCreatedAt3 = new \DateTime($createdAt3); + $parsedExpectedUpdatedAt3 = new \DateTime($updatedAt3); + $this->assertEquals($parsedExpectedCreatedAt3->getTimestamp(), $parsedCreatedAt3->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt3->getTimestamp(), $parsedUpdatedAt3->getTimestamp()); // updateDocument with preserved $updatedAt and custom date field $newCurDate1 = '2000-02-01T00:00:00.000+00:00'; @@ -1092,11 +1127,21 @@ public function testSchemalessDates(): void '$updatedAt' => $newUpdatedAt1, ])); }); - $this->assertEquals($newCurDate1, $updated1->getAttribute('curDate')); - $this->assertEquals($newUpdatedAt1, $updated1->getAttribute('$updatedAt')); + $updatedCurDate1 = $updated1->getAttribute('curDate'); + $updatedUpdatedAt1 = $updated1->getAttribute('$updatedAt'); + $parsedUpdatedCurDate1 = new \DateTime($updatedCurDate1); + $parsedUpdatedUpdatedAt1 = new \DateTime($updatedUpdatedAt1); + $parsedExpectedNewCurDate1 = new \DateTime($newCurDate1); + $parsedExpectedNewUpdatedAt1 = new \DateTime($newUpdatedAt1); + $this->assertEquals($parsedExpectedNewCurDate1->getTimestamp(), $parsedUpdatedCurDate1->getTimestamp()); + $this->assertEquals($parsedExpectedNewUpdatedAt1->getTimestamp(), $parsedUpdatedUpdatedAt1->getTimestamp()); $refetched1 = $database->getDocument($col, 'd1'); - $this->assertEquals($newCurDate1, $refetched1->getAttribute('curDate')); - $this->assertEquals($newUpdatedAt1, $refetched1->getAttribute('$updatedAt')); + $refetchedCurDate1 = $refetched1->getAttribute('curDate'); + $refetchedUpdatedAt1 = $refetched1->getAttribute('$updatedAt'); + $parsedRefetchedCurDate1 = new \DateTime($refetchedCurDate1); + $parsedRefetchedUpdatedAt1 = new \DateTime($refetchedUpdatedAt1); + $this->assertEquals($parsedExpectedNewCurDate1->getTimestamp(), $parsedRefetchedCurDate1->getTimestamp()); + $this->assertEquals($parsedExpectedNewUpdatedAt1->getTimestamp(), $parsedRefetchedUpdatedAt1->getTimestamp()); // updateDocuments with preserved $updatedAt over a subset $bulkCurDate = '2001-01-01T00:00:00.000+00:00'; @@ -1114,10 +1159,20 @@ public function testSchemalessDates(): void $this->assertEquals(2, $updatedCount); $afterBulk2 = $database->getDocument($col, 'd2'); $afterBulk3 = $database->getDocument($col, 'd3'); - $this->assertEquals($bulkCurDate, $afterBulk2->getAttribute('curDate')); - $this->assertEquals($bulkUpdatedAt, $afterBulk2->getAttribute('$updatedAt')); - $this->assertEquals($bulkCurDate, $afterBulk3->getAttribute('curDate')); - $this->assertEquals($bulkUpdatedAt, $afterBulk3->getAttribute('$updatedAt')); + $bulkCurDate2 = $afterBulk2->getAttribute('curDate'); + $bulkUpdatedAt2 = $afterBulk2->getAttribute('$updatedAt'); + $bulkCurDate3 = $afterBulk3->getAttribute('curDate'); + $bulkUpdatedAt3 = $afterBulk3->getAttribute('$updatedAt'); + $parsedBulkCurDate2 = new \DateTime($bulkCurDate2); + $parsedBulkUpdatedAt2 = new \DateTime($bulkUpdatedAt2); + $parsedBulkCurDate3 = new \DateTime($bulkCurDate3); + $parsedBulkUpdatedAt3 = new \DateTime($bulkUpdatedAt3); + $parsedExpectedBulkCurDate = new \DateTime($bulkCurDate); + $parsedExpectedBulkUpdatedAt = new \DateTime($bulkUpdatedAt); + $this->assertEquals($parsedExpectedBulkCurDate->getTimestamp(), $parsedBulkCurDate2->getTimestamp()); + $this->assertEquals($parsedExpectedBulkUpdatedAt->getTimestamp(), $parsedBulkUpdatedAt2->getTimestamp()); + $this->assertEquals($parsedExpectedBulkCurDate->getTimestamp(), $parsedBulkCurDate3->getTimestamp()); + $this->assertEquals($parsedExpectedBulkUpdatedAt->getTimestamp(), $parsedBulkUpdatedAt3->getTimestamp()); // upsertDocument: create new then update existing with preserved dates $createdAt4 = '2003-03-03T03:03:03.000+00:00'; @@ -1133,9 +1188,18 @@ public function testSchemalessDates(): void ])); }); $this->assertEquals('d4', $up1->getId()); - $this->assertEquals($curDate4, $up1->getAttribute('curDate')); - $this->assertEquals($createdAt4, $up1->getAttribute('$createdAt')); - $this->assertEquals($updatedAt4, $up1->getAttribute('$updatedAt')); + $up1CurDate4 = $up1->getAttribute('curDate'); + $up1CreatedAt4 = $up1->getAttribute('$createdAt'); + $up1UpdatedAt4 = $up1->getAttribute('$updatedAt'); + $parsedUp1CurDate4 = new \DateTime($up1CurDate4); + $parsedUp1CreatedAt4 = new \DateTime($up1CreatedAt4); + $parsedUp1UpdatedAt4 = new \DateTime($up1UpdatedAt4); + $parsedExpectedCurDate4 = new \DateTime($curDate4); + $parsedExpectedCreatedAt4 = new \DateTime($createdAt4); + $parsedExpectedUpdatedAt4 = new \DateTime($updatedAt4); + $this->assertEquals($parsedExpectedCurDate4->getTimestamp(), $parsedUp1CurDate4->getTimestamp()); + $this->assertEquals($parsedExpectedCreatedAt4->getTimestamp(), $parsedUp1CreatedAt4->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt4->getTimestamp(), $parsedUp1UpdatedAt4->getTimestamp()); $updatedAt4b = '2003-03-06T06:06:06.000+00:00'; $curDate4b = '2003-03-07T07:07:07.000+00:00'; @@ -1146,11 +1210,21 @@ public function testSchemalessDates(): void '$updatedAt' => $updatedAt4b, ])); }); - $this->assertEquals($curDate4b, $up2->getAttribute('curDate')); - $this->assertEquals($updatedAt4b, $up2->getAttribute('$updatedAt')); + $up2CurDate4b = $up2->getAttribute('curDate'); + $up2UpdatedAt4b = $up2->getAttribute('$updatedAt'); + $parsedUp2CurDate4b = new \DateTime($up2CurDate4b); + $parsedUp2UpdatedAt4b = new \DateTime($up2UpdatedAt4b); + $parsedExpectedCurDate4b = new \DateTime($curDate4b); + $parsedExpectedUpdatedAt4b = new \DateTime($updatedAt4b); + $this->assertEquals($parsedExpectedCurDate4b->getTimestamp(), $parsedUp2CurDate4b->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt4b->getTimestamp(), $parsedUp2UpdatedAt4b->getTimestamp()); $refetched4 = $database->getDocument($col, 'd4'); - $this->assertEquals($curDate4b, $refetched4->getAttribute('curDate')); - $this->assertEquals($updatedAt4b, $refetched4->getAttribute('$updatedAt')); + $refetched4CurDate4b = $refetched4->getAttribute('curDate'); + $refetched4UpdatedAt4b = $refetched4->getAttribute('$updatedAt'); + $parsedRefetched4CurDate4b = new \DateTime($refetched4CurDate4b); + $parsedRefetched4UpdatedAt4b = new \DateTime($refetched4UpdatedAt4b); + $this->assertEquals($parsedExpectedCurDate4b->getTimestamp(), $parsedRefetched4CurDate4b->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt4b->getTimestamp(), $parsedRefetched4UpdatedAt4b->getTimestamp()); // upsertDocuments: mix create and update with preserved dates $createdAt5 = '2004-04-01T01:01:01.000+00:00'; @@ -1178,13 +1252,28 @@ public function testSchemalessDates(): void $this->assertEquals(2, $upCount); $fetched5 = $database->getDocument($col, 'd5'); - $this->assertEquals($curDate5, $fetched5->getAttribute('curDate')); - $this->assertEquals($createdAt5, $fetched5->getAttribute('$createdAt')); - $this->assertEquals($updatedAt5, $fetched5->getAttribute('$updatedAt')); + $fetched5CurDate5 = $fetched5->getAttribute('curDate'); + $fetched5CreatedAt5 = $fetched5->getAttribute('$createdAt'); + $fetched5UpdatedAt5 = $fetched5->getAttribute('$updatedAt'); + $parsedFetched5CurDate5 = new \DateTime($fetched5CurDate5); + $parsedFetched5CreatedAt5 = new \DateTime($fetched5CreatedAt5); + $parsedFetched5UpdatedAt5 = new \DateTime($fetched5UpdatedAt5); + $parsedExpectedCurDate5 = new \DateTime($curDate5); + $parsedExpectedCreatedAt5 = new \DateTime($createdAt5); + $parsedExpectedUpdatedAt5 = new \DateTime($updatedAt5); + $this->assertEquals($parsedExpectedCurDate5->getTimestamp(), $parsedFetched5CurDate5->getTimestamp()); + $this->assertEquals($parsedExpectedCreatedAt5->getTimestamp(), $parsedFetched5CreatedAt5->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt5->getTimestamp(), $parsedFetched5UpdatedAt5->getTimestamp()); $fetched2b = $database->getDocument($col, 'd2'); - $this->assertEquals($curDate2b, $fetched2b->getAttribute('curDate')); - $this->assertEquals($updatedAt2b, $fetched2b->getAttribute('$updatedAt')); + $fetched2bCurDate2b = $fetched2b->getAttribute('curDate'); + $fetched2bUpdatedAt2b = $fetched2b->getAttribute('$updatedAt'); + $parsedFetched2bCurDate2b = new \DateTime($fetched2bCurDate2b); + $parsedFetched2bUpdatedAt2b = new \DateTime($fetched2bUpdatedAt2b); + $parsedExpectedCurDate2b = new \DateTime($curDate2b); + $parsedExpectedUpdatedAt2b = new \DateTime($updatedAt2b); + $this->assertEquals($parsedExpectedCurDate2b->getTimestamp(), $parsedFetched2bCurDate2b->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt2b->getTimestamp(), $parsedFetched2bUpdatedAt2b->getTimestamp()); // increase/decrease should not affect date types; ensure they remain strings $afterInc = $database->increaseDocumentAttribute($col, 'd1', 'counter', 5); @@ -2128,4 +2217,534 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void $database->deleteCollection($col); } + + public function testSchemalessDatetimeCreationAndFetching(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_datetime'); + $database->createCollection($col); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + // Create documents with ISO 8601 datetime strings (20-40 chars) + // MongoDB converts ISO 8601 to UTCDateTime and returns in 'Y-m-d H:i:s.v' format + $datetime1 = '2024-01-15T10:30:00.000+00:00'; + $datetime2 = '2024-02-20T14:45:30.123Z'; + $datetime3 = '2024-03-25T08:15:45.000000+05:30'; + + $doc1 = $database->createDocument($col, new Document([ + '$id' => 'dt1', + '$permissions' => $permissions, + 'eventDate' => $datetime1, + 'name' => 'Event 1' + ])); + + $doc2 = $database->createDocument($col, new Document([ + '$id' => 'dt2', + '$permissions' => $permissions, + 'eventDate' => $datetime2, + 'name' => 'Event 2' + ])); + + $doc3 = $database->createDocument($col, new Document([ + '$id' => 'dt3', + '$permissions' => $permissions, + 'eventDate' => $datetime3, + 'name' => 'Event 3' + ])); + + // Verify creation - check that datetime is stored and returned as string + $this->assertEquals('dt1', $doc1->getId()); + $eventDate1 = $doc1->getAttribute('eventDate'); + $this->assertTrue(is_string($eventDate1)); + $this->assertGreaterThanOrEqual(20, strlen($eventDate1)); + $this->assertLessThanOrEqual(40, strlen($eventDate1)); + + // Fetch and verify - MongoDB returns datetime in 'Y-m-d H:i:s.v' format + $fetched1 = $database->getDocument($col, 'dt1'); + $fetchedEventDate1 = $fetched1->getAttribute('eventDate'); + $this->assertTrue(is_string($fetchedEventDate1)); + $this->assertEquals('Event 1', $fetched1->getAttribute('name')); + + // Verify datetime values are equivalent by parsing (MongoDB converts to UTC) + $parsedInput1 = new \DateTime($datetime1); + $parsedOutput1 = new \DateTime($fetchedEventDate1); + $this->assertEquals($parsedInput1->getTimestamp(), $parsedOutput1->getTimestamp()); + + $fetched2 = $database->getDocument($col, 'dt2'); + $fetchedEventDate2 = $fetched2->getAttribute('eventDate'); + $this->assertTrue(is_string($fetchedEventDate2)); + $parsedInput2 = new \DateTime($datetime2); + $parsedOutput2 = new \DateTime($fetchedEventDate2); + $this->assertEquals($parsedInput2->getTimestamp(), $parsedOutput2->getTimestamp()); + + $fetched3 = $database->getDocument($col, 'dt3'); + $fetchedEventDate3 = $fetched3->getAttribute('eventDate'); + $this->assertTrue(is_string($fetchedEventDate3)); + // Verify it's a valid datetime string (format may vary slightly) + $this->assertGreaterThanOrEqual(20, strlen($fetchedEventDate3)); + $this->assertLessThanOrEqual(40, strlen($fetchedEventDate3)); + $parsedInput3 = new \DateTime($datetime3); + $parsedOutput3 = new \DateTime($fetchedEventDate3); + // MongoDB converts to UTC, so timestamps should match + $this->assertEquals($parsedInput3->getTimestamp(), $parsedOutput3->getTimestamp()); + + // Find all datetime documents + $allDocs = $database->find($col); + $this->assertCount(3, $allDocs); + + // Verify all documents are present + $allIds = array_map(fn ($doc) => $doc->getId(), $allDocs); + $this->assertContains('dt1', $allIds); + $this->assertContains('dt2', $allIds); + $this->assertContains('dt3', $allIds); + + // Update datetime + $newDatetime = '2024-12-31T23:59:59.999+00:00'; + $updated = $database->updateDocument($col, 'dt1', new Document([ + 'eventDate' => $newDatetime + ])); + $updatedEventDate = $updated->getAttribute('eventDate'); + $this->assertTrue(is_string($updatedEventDate)); + $this->assertGreaterThanOrEqual(20, strlen($updatedEventDate)); + $this->assertLessThanOrEqual(40, strlen($updatedEventDate)); + + $refetched = $database->getDocument($col, 'dt1'); + $refetchedEventDate = $refetched->getAttribute('eventDate'); + $this->assertTrue(is_string($refetchedEventDate)); + $parsedNewInput = new \DateTime($newDatetime); + $parsedNewOutput = new \DateTime($refetchedEventDate); + $this->assertEquals($parsedNewInput->getTimestamp(), $parsedNewOutput->getTimestamp()); + + $database->deleteCollection($col); + } + + public function testSchemalessTTLExpiry(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + if (!$database->getAdapter()->getSupportTTLIndexes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl_expiry'); + $database->createCollection($col); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + // Create TTL index with 60 seconds expiry + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expiresAt', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 10 + ) + ); + + $now = new \DateTime(); + $expiredTime = (clone $now)->modify('-10 seconds'); // Already expired + $futureTime = (clone $now)->modify('+120 seconds'); // Will expire in 2 minutes + + // Create mixed documents: some with expiresAt, some without + $doc1 = $database->createDocument($col, new Document([ + '$id' => 'expired_doc', + '$permissions' => $permissions, + 'expiresAt' => $expiredTime->format(\DateTime::ATOM), + 'data' => 'This should expire', + 'type' => 'temporary' + ])); + + $doc2 = $database->createDocument($col, new Document([ + '$id' => 'future_doc', + '$permissions' => $permissions, + 'expiresAt' => $futureTime->format(\DateTime::ATOM), + 'data' => 'This should not expire yet', + 'type' => 'temporary' + ])); + + $doc3 = $database->createDocument($col, new Document([ + '$id' => 'permanent_doc', + '$permissions' => $permissions, + 'data' => 'This should never expire', + 'type' => 'permanent' + ])); + + $doc4 = $database->createDocument($col, new Document([ + '$id' => 'another_permanent', + '$permissions' => $permissions, + 'data' => 'This should also never expire', + 'type' => 'permanent' + ])); + + // Verify all documents were created + $this->assertEquals('expired_doc', $doc1->getId()); + $this->assertEquals('future_doc', $doc2->getId()); + $this->assertEquals('permanent_doc', $doc3->getId()); + $this->assertEquals('another_permanent', $doc4->getId()); + + // Initial count should be 4 + $initialDocs = $database->find($col); + $this->assertCount(4, $initialDocs); + + // Wait for TTL to expire (at least 60 seconds + buffer) + // Note: MongoDB TTL cleanup runs every 60 seconds, so we need to wait + sleep(65); + + // Fetch collection to trigger TTL cleanup check + $collection = $database->getCollection($col); + $this->assertNotNull($collection); + + // After expiry, expired document should be gone + // Documents without expiresAt should remain + $remainingDocs = $database->find($col); + $remainingIds = array_map(fn ($doc) => $doc->getId(), $remainingDocs); + + // The expired document should be deleted + $this->assertNotContains('expired_doc', $remainingIds); + + // Documents without expiresAt should still exist + $this->assertContains('permanent_doc', $remainingIds); + $this->assertContains('another_permanent', $remainingIds); + + // Future document might still exist (depending on timing) or might be deleted + // The key test is that permanent docs remain and expired doc is gone + $this->assertGreaterThanOrEqual(2, count($remainingDocs)); // At least 2 permanent docs + $this->assertLessThanOrEqual(3, count($remainingDocs)); // At most 3 (if future doc still exists) + + // Verify permanent documents are still accessible + $permanent1 = $database->getDocument($col, 'permanent_doc'); + $this->assertFalse($permanent1->isEmpty()); + $this->assertEquals('permanent', $permanent1->getAttribute('type')); + + $permanent2 = $database->getDocument($col, 'another_permanent'); + $this->assertFalse($permanent2->isEmpty()); + $this->assertEquals('permanent', $permanent2->getAttribute('type')); + + // Verify expired document is gone + $expired = $database->getDocument($col, 'expired_doc'); + $this->assertTrue($expired->isEmpty()); + + $database->deleteCollection($col); + } + + public function testStringAndDatetime(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_str_datetime'); + $database->createCollection($col); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + // Create documents with mix of formatted dates (ISO 8601) and non-formatted dates (regular strings) + // Each document has two fields: str and datetime + $docs = [ + new Document([ + '$id' => 'doc1', + '$permissions' => $permissions, + 'str' => '2024-01-15T10:30:00.000+00:00', // ISO 8601 formatted date as string + 'datetime' => '2024-01-15T10:30:00.000+00:00' // ISO 8601 formatted date + ]), + new Document([ + '$id' => 'doc2', + '$permissions' => $permissions, + 'str' => 'just a regular string', // Non-formatted string + 'datetime' => '2024-02-20T14:45:30.123Z' // ISO 8601 formatted date + ]), + new Document([ + '$id' => 'doc3', + '$permissions' => $permissions, + 'str' => '2024-03-25T08:15:45.000000+05:30', // ISO 8601 formatted date as string + 'datetime' => 'not a date string' // Non-formatted string in datetime field + ]), + new Document([ + '$id' => 'doc4', + '$permissions' => $permissions, + 'str' => 'another string value', + 'datetime' => '2024-12-31T23:59:59.999+00:00' // ISO 8601 formatted date + ]), + new Document([ + '$id' => 'doc5', + '$permissions' => $permissions, + 'str' => '2024-06-15T12:00:00.000Z', // ISO 8601 formatted date as string + 'datetime' => '2024-06-15T12:00:00.000Z' // ISO 8601 formatted date + ]), + ]; + + $createdCount = $database->createDocuments($col, $docs); + $this->assertEquals(5, $createdCount); + + // Fetch all documents and validate + $allDocs = $database->find($col); + $this->assertCount(5, $allDocs); + + // Validate each document + $doc1 = $database->getDocument($col, 'doc1'); + $this->assertEquals('doc1', $doc1->getId()); + $this->assertTrue(is_string($doc1->getAttribute('str'))); + $this->assertTrue(is_string($doc1->getAttribute('datetime'))); + // str field should remain as string (even if it looks like a date) + $this->assertGreaterThanOrEqual(20, strlen($doc1->getAttribute('str'))); + $this->assertLessThanOrEqual(40, strlen($doc1->getAttribute('str'))); + // datetime field should be converted to MongoDB format if it's a valid ISO date + $datetime1 = $doc1->getAttribute('datetime'); + $this->assertTrue(is_string($datetime1)); + $this->assertGreaterThanOrEqual(20, strlen($datetime1)); + $this->assertLessThanOrEqual(40, strlen($datetime1)); + // Verify it's a valid datetime by parsing + $parsed1 = new \DateTime($datetime1); + $this->assertInstanceOf(\DateTime::class, $parsed1); + + $doc2 = $database->getDocument($col, 'doc2'); + $this->assertEquals('doc2', $doc2->getId()); + $this->assertEquals('just a regular string', $doc2->getAttribute('str')); + $datetime2 = $doc2->getAttribute('datetime'); + $this->assertTrue(is_string($datetime2)); + $parsed2 = new \DateTime($datetime2); + $this->assertInstanceOf(\DateTime::class, $parsed2); + + $doc3 = $database->getDocument($col, 'doc3'); + $this->assertEquals('doc3', $doc3->getId()); + $str3 = $doc3->getAttribute('str'); + $this->assertTrue(is_string($str3)); + $this->assertGreaterThanOrEqual(20, strlen($str3)); + $this->assertLessThanOrEqual(40, strlen($str3)); + // datetime field contains non-date string, should remain as string + $datetime3 = $doc3->getAttribute('datetime'); + $this->assertEquals('not a date string', $datetime3); + $this->assertTrue(is_string($datetime3)); + + $doc4 = $database->getDocument($col, 'doc4'); + $this->assertEquals('doc4', $doc4->getId()); + $this->assertEquals('another string value', $doc4->getAttribute('str')); + $datetime4 = $doc4->getAttribute('datetime'); + $this->assertTrue(is_string($datetime4)); + $parsed4 = new \DateTime($datetime4); + $this->assertInstanceOf(\DateTime::class, $parsed4); + + $doc5 = $database->getDocument($col, 'doc5'); + $this->assertEquals('doc5', $doc5->getId()); + $str5 = $doc5->getAttribute('str'); + $this->assertTrue(is_string($str5)); + $this->assertGreaterThanOrEqual(20, strlen($str5)); + $this->assertLessThanOrEqual(40, strlen($str5)); + $datetime5 = $doc5->getAttribute('datetime'); + $this->assertTrue(is_string($datetime5)); + $parsed5 = new \DateTime($datetime5); + $this->assertInstanceOf(\DateTime::class, $parsed5); + + // Verify all documents are present using simple find + $allDocs = $database->find($col); + $this->assertCount(5, $allDocs); + $allIds = array_map(fn ($doc) => $doc->getId(), $allDocs); + $this->assertContains('doc1', $allIds); + $this->assertContains('doc2', $allIds); + $this->assertContains('doc3', $allIds); + $this->assertContains('doc4', $allIds); + $this->assertContains('doc5', $allIds); + + $database->deleteCollection($col); + } + + public function testStringAndDateWithTTL(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + if (!$database->getAdapter()->getSupportTTLIndexes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_str_date_ttl'); + $database->createCollection($col); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + // Create TTL index on expiresAt field + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expiresAt', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 10 + ) + ); + + $now = new \DateTime(); + $expiredTime = (clone $now)->modify('-10 seconds'); // Already expired + $futureTime = (clone $now)->modify('+120 seconds'); // Will expire in 2 minutes + + // Create documents with mix of datetime values and random strings in expiresAt + $docs = [ + new Document([ + '$id' => 'doc_datetime_expired', + '$permissions' => $permissions, + 'expiresAt' => $expiredTime->format(\DateTime::ATOM), // Valid datetime - should expire + 'data' => 'This should expire', + 'type' => 'datetime' + ]), + new Document([ + '$id' => 'doc_datetime_future', + '$permissions' => $permissions, + 'expiresAt' => $futureTime->format(\DateTime::ATOM), // Valid datetime - future + 'data' => 'This should not expire yet', + 'type' => 'datetime' + ]), + new Document([ + '$id' => 'doc_string_random', + '$permissions' => $permissions, + 'expiresAt' => 'random_string_value_12345', // Random string - should not expire + 'data' => 'This should never expire', + 'type' => 'string' + ]), + new Document([ + '$id' => 'doc_string_another', + '$permissions' => $permissions, + 'expiresAt' => 'another_random_string_xyz', // Random string - should not expire + 'data' => 'This should also never expire', + 'type' => 'string' + ]), + new Document([ + '$id' => 'doc_datetime_valid', + '$permissions' => $permissions, + 'expiresAt' => $futureTime->format(\DateTime::ATOM), // Valid datetime - future + 'data' => 'This is a valid datetime', + 'type' => 'datetime' + ]), + ]; + + $createdCount = $database->createDocuments($col, $docs); + $this->assertEquals(5, $createdCount); + + // Verify all documents were created + $initialDocs = $database->find($col); + $this->assertCount(5, $initialDocs); + + // Verify documents with datetime values + $docDatetimeExpired = $database->getDocument($col, 'doc_datetime_expired'); + $this->assertFalse($docDatetimeExpired->isEmpty()); + $expiresAt1 = $docDatetimeExpired->getAttribute('expiresAt'); + $this->assertTrue(is_string($expiresAt1)); + // Should be converted to MongoDB datetime format if it was a valid ISO date + $this->assertGreaterThanOrEqual(20, strlen($expiresAt1)); + $this->assertLessThanOrEqual(40, strlen($expiresAt1)); + // Verify it can be parsed as datetime + $parsed1 = new \DateTime($expiresAt1); + $this->assertInstanceOf(\DateTime::class, $parsed1); + + $docDatetimeFuture = $database->getDocument($col, 'doc_datetime_future'); + $this->assertFalse($docDatetimeFuture->isEmpty()); + $expiresAt2 = $docDatetimeFuture->getAttribute('expiresAt'); + $this->assertTrue(is_string($expiresAt2)); + $parsed2 = new \DateTime($expiresAt2); + $this->assertInstanceOf(\DateTime::class, $parsed2); + + // Verify documents with random strings remain as strings + $docStringRandom = $database->getDocument($col, 'doc_string_random'); + $this->assertFalse($docStringRandom->isEmpty()); + $expiresAt3 = $docStringRandom->getAttribute('expiresAt'); + $this->assertEquals('random_string_value_12345', $expiresAt3); + $this->assertTrue(is_string($expiresAt3)); + // Should remain as the original string (not converted to datetime) + $this->assertEquals('random_string_value_12345', $expiresAt3); + + $docStringAnother = $database->getDocument($col, 'doc_string_another'); + $this->assertFalse($docStringAnother->isEmpty()); + $expiresAt4 = $docStringAnother->getAttribute('expiresAt'); + $this->assertEquals('another_random_string_xyz', $expiresAt4); + $this->assertTrue(is_string($expiresAt4)); + + // Wait for TTL to expire (at least 60 seconds + buffer) + sleep(65); + + // Fetch collection to trigger TTL cleanup check + $collection = $database->getCollection($col); + $this->assertNotNull($collection); + + // After expiry, check remaining documents + $remainingDocs = $database->find($col); + $remainingIds = array_map(fn ($doc) => $doc->getId(), $remainingDocs); + + // The expired datetime document should be deleted + $this->assertNotContains('doc_datetime_expired', $remainingIds); + + // Documents with random strings should still exist (TTL doesn't affect non-datetime values) + $this->assertContains('doc_string_random', $remainingIds); + $this->assertContains('doc_string_another', $remainingIds); + + // Verify random string documents are still accessible with original string values + $remainingStringDoc1 = $database->getDocument($col, 'doc_string_random'); + $this->assertFalse($remainingStringDoc1->isEmpty()); + $this->assertEquals('random_string_value_12345', $remainingStringDoc1->getAttribute('expiresAt')); + $this->assertEquals('string', $remainingStringDoc1->getAttribute('type')); + + $remainingStringDoc2 = $database->getDocument($col, 'doc_string_another'); + $this->assertFalse($remainingStringDoc2->isEmpty()); + $this->assertEquals('another_random_string_xyz', $remainingStringDoc2->getAttribute('expiresAt')); + $this->assertEquals('string', $remainingStringDoc2->getAttribute('type')); + + // Verify expired datetime document is gone + $expiredDoc = $database->getDocument($col, 'doc_datetime_expired'); + $this->assertTrue($expiredDoc->isEmpty()); + + // Future datetime documents might still exist or be deleted depending on timing + // But at minimum, we should have the string documents + $this->assertGreaterThanOrEqual(2, count($remainingDocs)); // At least 2 string docs + $this->assertLessThanOrEqual(4, count($remainingDocs)); // At most 4 (if future datetime docs still exist) + + $database->deleteCollection($col); + } } From 9a21ce2de5d9ee772add6c4cc3993a6448292829 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k <83803257+ArnabChatterjee20k@users.noreply.github.com> Date: Tue, 13 Jan 2026 21:04:34 +0530 Subject: [PATCH 08/16] Update src/Database/Validator/Index.php Co-authored-by: Jake Barnby --- src/Database/Validator/Index.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php index 1e367d1ad..8b3d13cf2 100644 --- a/src/Database/Validator/Index.php +++ b/src/Database/Validator/Index.php @@ -753,7 +753,7 @@ public function checkTTLIndexes(Document $index): bool } if (count($attributes) !== 1) { - $this->message = 'TTL index can be created on a single datetime attribute'; + $this->message = 'TTL indexes must be created on a single datetime attribute.'; return false; } From 6fd965da02ddef3da4bd233f204636babacb074a Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 13 Jan 2026 21:14:27 +0530 Subject: [PATCH 09/16] fix: update TTL index default value to 1 and adjust related methods for consistency --- src/Database/Adapter.php | 4 +-- src/Database/Adapter/MariaDB.php | 4 +-- src/Database/Adapter/Mongo.php | 32 ++++++++++++++++---- src/Database/Adapter/MySQL.php | 2 +- src/Database/Adapter/Pool.php | 4 +-- src/Database/Adapter/Postgres.php | 4 +-- src/Database/Adapter/SQLite.php | 4 +-- src/Database/Database.php | 8 ++--- src/Database/Mirror.php | 2 +- src/Database/Validator/Index.php | 7 +---- tests/e2e/Adapter/Scopes/IndexTests.php | 4 +-- tests/e2e/Adapter/Scopes/SchemalessTests.php | 4 +-- tests/unit/Validator/IndexTest.php | 16 ++-------- 13 files changed, 49 insertions(+), 46 deletions(-) diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 37772e1b2..3b010ad89 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -671,7 +671,7 @@ abstract public function renameIndex(string $collection, string $old, string $ne * * @return bool */ - abstract public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool; + abstract public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool; /** * Delete Index @@ -1491,7 +1491,7 @@ public function getSupportForRegex(): bool * * @return bool */ - public function getSupportTTLIndexes(): bool + public function getSupportForTTLIndexes(): bool { return false; } diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index 242b0d9ad..6ae88336f 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -715,7 +715,7 @@ public function renameIndex(string $collection, string $old, string $new): bool * @return bool * @throws DatabaseException */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool { $metadataCollection = new Document(['$id' => Database::METADATA]); $collection = $this->getDocument($metadataCollection, $collection); @@ -2256,7 +2256,7 @@ public function getSupportForPOSIXRegex(): bool return false; } - public function getSupportTTLIndexes(): bool + public function getSupportForTTLIndexes(): bool { return false; } diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index ecb050f8e..2a875eb33 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -488,7 +488,7 @@ public function createCollection(string $name, array $attributes = [], array $in $orders = $index->getAttribute('orders'); // If sharedTables, always add _tenant as the first key - if ($this->sharedTables && $index->getAttribute('type') !== Database::INDEX_TTL) { + if ($this->shouldAddTenantToIndex($index)) { $key['_tenant'] = $this->getOrder(Database::ORDER_ASC); } @@ -916,7 +916,7 @@ public function deleteRelationship( * @return bool * @throws Exception */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool { $name = $this->getNamespace() . '_' . $this->filter($collection); $id = $this->filter($id); @@ -925,7 +925,7 @@ public function createIndex(string $collection, string $id, string $type, array $indexes['name'] = $id; // If sharedTables, always add _tenant as the first key - if ($this->sharedTables && $type !== Database::INDEX_TTL) { + if ($this->shouldAddTenantToIndex($type)) { $indexes['key']['_tenant'] = $this->getOrder(Database::ORDER_ASC); } @@ -2668,6 +2668,25 @@ protected function getOrder(string $order): int }; } + /** + * Check if tenant should be added to index + * + * @param Document|string $indexOrType Index document or index type string + * @return bool + */ + protected function shouldAddTenantToIndex(Document|string $indexOrType): bool + { + if (!$this->sharedTables) { + return false; + } + + $indexType = $indexOrType instanceof Document + ? $indexOrType->getAttribute('type') + : $indexOrType; + + return $indexType !== Database::INDEX_TTL; + } + /** * @param array $selections * @param string $prefix @@ -3412,7 +3431,7 @@ public function getSupportForTrigramIndex(): bool return false; } - public function getSupportTTLIndexes(): bool + public function getSupportForTTLIndexes(): bool { return true; } @@ -3461,10 +3480,11 @@ protected function isExtendedISODatetime(string $val): bool return false; } - if ($hasZ && $len > 26) { + if ($hasOffset && $len > 31) { return false; } - if ($hasOffset && ($len < 25 || $len > 31)) { + + if ($hasZ && $len > 26) { return false; } diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index d5740cf66..a0294902a 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -318,7 +318,7 @@ protected function getOperatorSQL(string $column, \Utopia\Database\Operator $ope return parent::getOperatorSQL($column, $operator, $bindIndex); } - public function getSupportTTLIndexes(): bool + public function getSupportForTTLIndexes(): bool { return false; } diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index 1d8c892e6..9a42ce3cc 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -210,7 +210,7 @@ public function renameIndex(string $collection, string $old, string $new): bool return $this->delegate(__FUNCTION__, \func_get_args()); } - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool { return $this->delegate(__FUNCTION__, \func_get_args()); } @@ -650,7 +650,7 @@ public function getSupportForAlterLocks(): bool return $this->delegate(__FUNCTION__, \func_get_args()); } - public function getSupportTTLIndexes(): bool + public function getSupportForTTLIndexes(): bool { return $this->delegate(__FUNCTION__, \func_get_args()); } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 5cfe39d55..a6f701b0d 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -879,7 +879,7 @@ public function deleteRelationship( * @return bool */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool { $collection = $this->filter($collection); $id = $this->filter($id); @@ -2833,7 +2833,7 @@ protected function getSQLTable(string $name): string return "{$this->quote($this->getDatabase())}.{$this->quote($table)}"; } - public function getSupportTTLIndexes(): bool + public function getSupportForTTLIndexes(): bool { return false; } diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 55e195dc2..0e9d1ba6f 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -456,7 +456,7 @@ public function renameIndex(string $collection, string $old, string $new): bool * @throws Exception * @throws PDOException */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 0): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool { $name = $this->filter($collection); $id = $this->filter($id); @@ -1910,7 +1910,7 @@ public function getSupportForPOSIXRegex(): bool return false; } - public function getSupportTTLIndexes(): bool + public function getSupportForTTLIndexes(): bool { return false; } diff --git a/src/Database/Database.php b/src/Database/Database.php index 65526c348..b4078bd99 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1649,7 +1649,7 @@ public function createCollection(string $id, array $attributes = [], array $inde $this->adapter->getSupportForIndex(), $this->adapter->getSupportForUniqueIndex(), $this->adapter->getSupportForFulltextIndex(), - $this->adapter->getSupportTTLIndexes() + $this->adapter->getSupportForTTLIndexes() ); foreach ($indexes as $index) { if (!$validator->isValid($index)) { @@ -2800,7 +2800,7 @@ public function updateAttribute(string $collection, string $id, ?string $type = $this->adapter->getSupportForIndex(), $this->adapter->getSupportForUniqueIndex(), $this->adapter->getSupportForFulltextIndex(), - $this->adapter->getSupportTTLIndexes() + $this->adapter->getSupportForTTLIndexes() ); foreach ($indexes as $index) { @@ -3617,7 +3617,7 @@ public function renameIndex(string $collection, string $old, string $new): bool * @throws StructureException * @throws Exception */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = [], int $ttl = 0): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = [], int $ttl = 1): bool { if (empty($attributes)) { throw new DatabaseException('Missing attributes'); @@ -3697,7 +3697,7 @@ public function createIndex(string $collection, string $id, string $type, array $this->adapter->getSupportForIndex(), $this->adapter->getSupportForUniqueIndex(), $this->adapter->getSupportForFulltextIndex(), - $this->adapter->getSupportTTLIndexes() + $this->adapter->getSupportForTTLIndexes() ); if (!$validator->isValid($index)) { throw new IndexException($validator->getDescription()); diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php index 3c65a88a9..5944e4aa5 100644 --- a/src/Database/Mirror.php +++ b/src/Database/Mirror.php @@ -469,7 +469,7 @@ public function deleteAttribute(string $collection, string $id): bool return $result; } - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = [], int $ttl = 0): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = [], int $ttl = 1): bool { $result = $this->source->createIndex($collection, $id, $type, $attributes, $lengths, $orders, $ttl); diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php index 1e367d1ad..a86e55ec5 100644 --- a/src/Database/Validator/Index.php +++ b/src/Database/Validator/Index.php @@ -753,12 +753,7 @@ public function checkTTLIndexes(Document $index): bool } if (count($attributes) !== 1) { - $this->message = 'TTL index can be created on a single datetime attribute'; - return false; - } - - if (empty($orders)) { - $this->message = 'TTL index need explicit orders. Add the orders to create this index.'; + $this->message = 'TTL indexes must be created on a single datetime attribute'; return false; } diff --git a/tests/e2e/Adapter/Scopes/IndexTests.php b/tests/e2e/Adapter/Scopes/IndexTests.php index 6c5e21aaa..bd5386786 100644 --- a/tests/e2e/Adapter/Scopes/IndexTests.php +++ b/tests/e2e/Adapter/Scopes/IndexTests.php @@ -790,7 +790,7 @@ public function testTTLIndexes(): void /** @var Database $database */ $database = static::getDatabase(); - if (!$database->getAdapter()->getSupportTTLIndexes()) { + if (!$database->getAdapter()->getSupportForTTLIndexes()) { $this->expectNotToPerformAssertions(); return; } @@ -904,7 +904,7 @@ public function testTTLIndexDuplicatePrevention(): void /** @var Database $database */ $database = static::getDatabase(); - if (!$database->getAdapter()->getSupportTTLIndexes()) { + if (!$database->getAdapter()->getSupportForTTLIndexes()) { $this->expectNotToPerformAssertions(); return; } diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php index 9bac0d8fe..b8181bf6e 100644 --- a/tests/e2e/Adapter/Scopes/SchemalessTests.php +++ b/tests/e2e/Adapter/Scopes/SchemalessTests.php @@ -2341,7 +2341,7 @@ public function testSchemalessTTLExpiry(): void return; } - if (!$database->getAdapter()->getSupportTTLIndexes()) { + if (!$database->getAdapter()->getSupportForTTLIndexes()) { $this->expectNotToPerformAssertions(); return; } @@ -2595,7 +2595,7 @@ public function testStringAndDateWithTTL(): void return; } - if (!$database->getAdapter()->getSupportTTLIndexes()) { + if (!$database->getAdapter()->getSupportForTTLIndexes()) { $this->expectNotToPerformAssertions(); return; } diff --git a/tests/unit/Validator/IndexTest.php b/tests/unit/Validator/IndexTest.php index 664874051..1722c7301 100644 --- a/tests/unit/Validator/IndexTest.php +++ b/tests/unit/Validator/IndexTest.php @@ -665,7 +665,7 @@ public function testTTLIndexValidation(): void ]); $this->assertTrue($validator->isValid($validIndex)); - // Invalid: TTL index with TTL = 0 + // Invalid: TTL index with ttl = 1 $invalidIndexZero = new Document([ '$id' => ID::custom('idx_ttl_zero'), 'type' => Database::INDEX_TTL, @@ -711,19 +711,7 @@ public function testTTLIndexValidation(): void 'ttl' => 3600, ]); $this->assertFalse($validator->isValid($invalidIndexMulti)); - $this->assertStringContainsString('TTL index can be created on a single datetime attribute', $validator->getDescription()); - - // Invalid: TTL index without orders - $invalidIndexNoOrders = new Document([ - '$id' => ID::custom('idx_ttl_no_orders'), - 'type' => Database::INDEX_TTL, - 'attributes' => ['expiresAt'], - 'lengths' => [], - 'orders' => [], - 'ttl' => 3600, - ]); - $this->assertFalse($validator->isValid($invalidIndexNoOrders)); - $this->assertEquals('TTL index need explicit orders. Add the orders to create this index.', $validator->getDescription()); + $this->assertStringContainsString('TTL indexes must be created on a single datetime attribute', $validator->getDescription()); // Valid: TTL index with minimum valid TTL (1 second) $validIndexMin = new Document([ From affc6a1ff2838623600528224df4639f192555ad Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 13 Jan 2026 21:17:05 +0530 Subject: [PATCH 10/16] test: implement retry logic for TTL expiration checks in SchemalessTests --- tests/e2e/Adapter/Scopes/SchemalessTests.php | 67 +++++++++++++++----- 1 file changed, 52 insertions(+), 15 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php index b8181bf6e..b6e086759 100644 --- a/tests/e2e/Adapter/Scopes/SchemalessTests.php +++ b/tests/e2e/Adapter/Scopes/SchemalessTests.php @@ -2414,14 +2414,32 @@ public function testSchemalessTTLExpiry(): void $initialDocs = $database->find($col); $this->assertCount(4, $initialDocs); - // Wait for TTL to expire (at least 60 seconds + buffer) - // Note: MongoDB TTL cleanup runs every 60 seconds, so we need to wait - sleep(65); - - // Fetch collection to trigger TTL cleanup check - $collection = $database->getCollection($col); - $this->assertNotNull($collection); - + // Wait for TTL to expire with retry loop + // Note: MongoDB TTL cleanup runs every 60 seconds, so we need to retry + $maxRetries = 15; // 15 retries * 5 seconds = 75 seconds max + $retryDelay = 5; // Wait 5 seconds between retries + $expiredDocDeleted = false; + + for ($i = 0; $i < $maxRetries; $i++) { + sleep($retryDelay); + + // Fetch collection to trigger TTL cleanup check + $collection = $database->getCollection($col); + $this->assertNotNull($collection); + + // Check if expired document is gone + $remainingDocs = $database->find($col); + $remainingIds = array_map(fn ($doc) => $doc->getId(), $remainingDocs); + + if (!in_array('expired_doc', $remainingIds)) { + $expiredDocDeleted = true; + break; + } + } + + // Assert that expired document was deleted + $this->assertTrue($expiredDocDeleted, 'Expired document should have been deleted after TTL expiry'); + // After expiry, expired document should be gone // Documents without expiresAt should remain $remainingDocs = $database->find($col); @@ -2707,13 +2725,32 @@ public function testStringAndDateWithTTL(): void $this->assertEquals('another_random_string_xyz', $expiresAt4); $this->assertTrue(is_string($expiresAt4)); - // Wait for TTL to expire (at least 60 seconds + buffer) - sleep(65); - - // Fetch collection to trigger TTL cleanup check - $collection = $database->getCollection($col); - $this->assertNotNull($collection); - + // Wait for TTL to expire with retry loop + // Note: MongoDB TTL cleanup runs every 60 seconds, so we need to retry + $maxRetries = 15; // 15 retries * 5 seconds = 75 seconds max + $retryDelay = 5; // Wait 5 seconds between retries + $expiredDocDeleted = false; + + for ($i = 0; $i < $maxRetries; $i++) { + sleep($retryDelay); + + // Fetch collection to trigger TTL cleanup check + $collection = $database->getCollection($col); + $this->assertNotNull($collection); + + // Check if expired datetime document is gone + $remainingDocs = $database->find($col); + $remainingIds = array_map(fn ($doc) => $doc->getId(), $remainingDocs); + + if (!in_array('doc_datetime_expired', $remainingIds)) { + $expiredDocDeleted = true; + break; + } + } + + // Assert that expired document was deleted + $this->assertTrue($expiredDocDeleted, 'Expired datetime document should have been deleted after TTL expiry'); + // After expiry, check remaining documents $remainingDocs = $database->find($col); $remainingIds = array_map(fn ($doc) => $doc->getId(), $remainingDocs); From 3cbc6f64f44f2e486048f3e3c67b809b7be9f0ba Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 13 Jan 2026 21:19:13 +0530 Subject: [PATCH 11/16] linting --- src/Database/Adapter/Mongo.php | 4 ++-- tests/e2e/Adapter/Scopes/SchemalessTests.php | 24 ++++++++++---------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index 2a875eb33..0b82a9e5a 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -2670,7 +2670,7 @@ protected function getOrder(string $order): int /** * Check if tenant should be added to index - * + * * @param Document|string $indexOrType Index document or index type string * @return bool */ @@ -3483,7 +3483,7 @@ protected function isExtendedISODatetime(string $val): bool if ($hasOffset && $len > 31) { return false; } - + if ($hasZ && $len > 26) { return false; } diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php index b6e086759..240a4a58a 100644 --- a/tests/e2e/Adapter/Scopes/SchemalessTests.php +++ b/tests/e2e/Adapter/Scopes/SchemalessTests.php @@ -2419,27 +2419,27 @@ public function testSchemalessTTLExpiry(): void $maxRetries = 15; // 15 retries * 5 seconds = 75 seconds max $retryDelay = 5; // Wait 5 seconds between retries $expiredDocDeleted = false; - + for ($i = 0; $i < $maxRetries; $i++) { sleep($retryDelay); - + // Fetch collection to trigger TTL cleanup check $collection = $database->getCollection($col); $this->assertNotNull($collection); - + // Check if expired document is gone $remainingDocs = $database->find($col); $remainingIds = array_map(fn ($doc) => $doc->getId(), $remainingDocs); - + if (!in_array('expired_doc', $remainingIds)) { $expiredDocDeleted = true; break; } } - + // Assert that expired document was deleted $this->assertTrue($expiredDocDeleted, 'Expired document should have been deleted after TTL expiry'); - + // After expiry, expired document should be gone // Documents without expiresAt should remain $remainingDocs = $database->find($col); @@ -2730,27 +2730,27 @@ public function testStringAndDateWithTTL(): void $maxRetries = 15; // 15 retries * 5 seconds = 75 seconds max $retryDelay = 5; // Wait 5 seconds between retries $expiredDocDeleted = false; - + for ($i = 0; $i < $maxRetries; $i++) { sleep($retryDelay); - + // Fetch collection to trigger TTL cleanup check $collection = $database->getCollection($col); $this->assertNotNull($collection); - + // Check if expired datetime document is gone $remainingDocs = $database->find($col); $remainingIds = array_map(fn ($doc) => $doc->getId(), $remainingDocs); - + if (!in_array('doc_datetime_expired', $remainingIds)) { $expiredDocDeleted = true; break; } } - + // Assert that expired document was deleted $this->assertTrue($expiredDocDeleted, 'Expired datetime document should have been deleted after TTL expiry'); - + // After expiry, check remaining documents $remainingDocs = $database->find($col); $remainingIds = array_map(fn ($doc) => $doc->getId(), $remainingDocs); From 41ea5cac39676e51c6ea1656fce48b093ff7cd25 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 13 Jan 2026 21:26:24 +0530 Subject: [PATCH 12/16] updated waiting in ttl test --- tests/e2e/Adapter/Scopes/SchemalessTests.php | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php index 240a4a58a..5a17c0d53 100644 --- a/tests/e2e/Adapter/Scopes/SchemalessTests.php +++ b/tests/e2e/Adapter/Scopes/SchemalessTests.php @@ -2725,10 +2725,8 @@ public function testStringAndDateWithTTL(): void $this->assertEquals('another_random_string_xyz', $expiresAt4); $this->assertTrue(is_string($expiresAt4)); - // Wait for TTL to expire with retry loop - // Note: MongoDB TTL cleanup runs every 60 seconds, so we need to retry - $maxRetries = 15; // 15 retries * 5 seconds = 75 seconds max - $retryDelay = 5; // Wait 5 seconds between retries + $maxRetries = 25; + $retryDelay = 5; $expiredDocDeleted = false; for ($i = 0; $i < $maxRetries; $i++) { From 06e8d087a81e47d9345ec9dad6f968f5ffd3f682 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Tue, 13 Jan 2026 21:38:45 +0530 Subject: [PATCH 13/16] updated expiry tests --- tests/e2e/Adapter/Scopes/SchemalessTests.php | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php index 5a17c0d53..1936742fd 100644 --- a/tests/e2e/Adapter/Scopes/SchemalessTests.php +++ b/tests/e2e/Adapter/Scopes/SchemalessTests.php @@ -2771,10 +2771,6 @@ public function testStringAndDateWithTTL(): void $this->assertEquals('another_random_string_xyz', $remainingStringDoc2->getAttribute('expiresAt')); $this->assertEquals('string', $remainingStringDoc2->getAttribute('type')); - // Verify expired datetime document is gone - $expiredDoc = $database->getDocument($col, 'doc_datetime_expired'); - $this->assertTrue($expiredDoc->isEmpty()); - // Future datetime documents might still exist or be deleted depending on timing // But at minimum, we should have the string documents $this->assertGreaterThanOrEqual(2, count($remainingDocs)); // At least 2 string docs From 4cb686343e9b35ad8142fa7af4656ffadecb8e8f Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 14 Jan 2026 13:03:18 +0530 Subject: [PATCH 14/16] enforced one ttl index per collection --- src/Database/Database.php | 8 +++++ src/Database/Validator/Index.php | 14 ++++----- tests/e2e/Adapter/Scopes/IndexTests.php | 32 +++++++++++--------- tests/e2e/Adapter/Scopes/SchemalessTests.php | 31 ++++++++++--------- tests/unit/Validator/IndexTest.php | 4 +-- 5 files changed, 50 insertions(+), 39 deletions(-) diff --git a/src/Database/Database.php b/src/Database/Database.php index b4078bd99..b19cd66c6 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -1585,6 +1585,14 @@ public function createCollection(string $id, array $attributes = [], array $inde throw new DuplicateException('Collection ' . $id . ' already exists'); } + // Enforce single TTL index per collection + if ($this->validate && $this->getAdapter()->getSupportForTTLIndexes()) { + $ttlIndexes = array_filter($indexes, fn (Document $idx) => $idx->getAttribute('type') === self::INDEX_TTL); + if (count($ttlIndexes) > 1) { + throw new IndexException('There can be only one TTL index in a collection'); + } + } + /** * Fix metadata index length & orders */ diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php index 8b3d13cf2..4261b9851 100644 --- a/src/Database/Validator/Index.php +++ b/src/Database/Validator/Index.php @@ -776,17 +776,15 @@ public function checkTTLIndexes(Document $index): bool return false; } + // Check if there's already a TTL index in this collection foreach ($this->indexes as $existingIndex) { - $existingAttributes = $existingIndex->getAttribute('attributes', []); - $existingOrders = $existingIndex->getAttribute('orders', []); - $existingType = $existingIndex->getAttribute('type', ''); - if ($this->supportForAttributes && $existingType !== Database::INDEX_TTL) { + if ($existingIndex->getId() === $index->getId()) { continue; } - $attributeAlreadyPresent = ($this->supportForAttributes && in_array($attribute->getId(), $existingAttributes)) || in_array($attributeName, $existingAttributes); - $ordersMatched = empty(array_diff($existingOrders, $orders)); - if ($attributeAlreadyPresent && $ordersMatched) { - $this->message = 'There is already an index with the same attributes and orders'; + + // Check if existing index is also a TTL index + if ($existingIndex->getAttribute('type') === Database::INDEX_TTL) { + $this->message = 'There can be only one TTL index in a collection'; return false; } } diff --git a/tests/e2e/Adapter/Scopes/IndexTests.php b/tests/e2e/Adapter/Scopes/IndexTests.php index bd5386786..5364a4e65 100644 --- a/tests/e2e/Adapter/Scopes/IndexTests.php +++ b/tests/e2e/Adapter/Scopes/IndexTests.php @@ -937,13 +937,13 @@ public function testTTLIndexDuplicatePrevention(): void [Database::ORDER_ASC], 7200 // 2 hours ); - $this->fail('Expected exception for duplicate TTL index on same attribute'); + $this->fail('Expected exception for creating a second TTL index in a collection'); } catch (Exception $e) { $this->assertInstanceOf(DatabaseException::class, $e); - $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); } - $this->assertTrue( + try { $database->createIndex( $col, 'idx_ttl_deleted', @@ -952,16 +952,20 @@ public function testTTLIndexDuplicatePrevention(): void [], [Database::ORDER_ASC], 86400 // 24 hours - ) - ); + ); + $this->fail('Expected exception for creating a second TTL index in a collection'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); + } $collection = $database->getCollection($col); $indexes = $collection->getAttribute('indexes'); - $this->assertCount(2, $indexes); + $this->assertCount(1, $indexes); $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); $this->assertContains('idx_ttl_expires', $indexIds); - $this->assertContains('idx_ttl_deleted', $indexIds); + $this->assertNotContains('idx_ttl_deleted', $indexIds); try { $database->createIndex( @@ -973,10 +977,10 @@ public function testTTLIndexDuplicatePrevention(): void [Database::ORDER_ASC], 172800 // 48 hours ); - $this->fail('Expected exception for duplicate TTL index on same attribute'); + $this->fail('Expected exception for creating a second TTL index in a collection'); } catch (Exception $e) { $this->assertInstanceOf(DatabaseException::class, $e); - $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); } $this->assertTrue($database->deleteIndex($col, 'idx_ttl_expires')); @@ -984,9 +988,9 @@ public function testTTLIndexDuplicatePrevention(): void $this->assertTrue( $database->createIndex( $col, - 'idx_ttl_expires_new', + 'idx_ttl_deleted', Database::INDEX_TTL, - ['expiresAt'], + ['deletedAt'], [], [Database::ORDER_ASC], 1800 // 30 minutes @@ -995,11 +999,10 @@ public function testTTLIndexDuplicatePrevention(): void $collection = $database->getCollection($col); $indexes = $collection->getAttribute('indexes'); - $this->assertCount(2, $indexes); + $this->assertCount(1, $indexes); $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); $this->assertNotContains('idx_ttl_expires', $indexIds); - $this->assertContains('idx_ttl_expires_new', $indexIds); $this->assertContains('idx_ttl_deleted', $indexIds); $col3 = uniqid('sl_ttl_dup_collection'); @@ -1038,8 +1041,7 @@ public function testTTLIndexDuplicatePrevention(): void $this->fail('Expected exception for duplicate TTL indexes in createCollection'); } catch (Exception $e) { $this->assertInstanceOf(DatabaseException::class, $e); - // raised in the mongo level - $this->assertStringContainsString('Index already exists', $e->getMessage()); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); } // Cleanup diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php index 1936742fd..5fbb69e16 100644 --- a/tests/e2e/Adapter/Scopes/SchemalessTests.php +++ b/tests/e2e/Adapter/Scopes/SchemalessTests.php @@ -2111,13 +2111,13 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void [Database::ORDER_ASC], 7200 // 2 hours ); - $this->fail('Expected exception for duplicate TTL index on same attribute'); + $this->fail('Expected exception for creating a second TTL index in a collection'); } catch (Exception $e) { $this->assertInstanceOf(DatabaseException::class, $e); - $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); } - $this->assertTrue( + try { $database->createIndex( $col, 'idx_ttl_deleted', @@ -2126,16 +2126,20 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void [], [Database::ORDER_ASC], 86400 // 24 hours - ) - ); + ); + $this->fail('Expected exception for creating a second TTL index in a collection'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); + } $collection = $database->getCollection($col); $indexes = $collection->getAttribute('indexes'); - $this->assertCount(2, $indexes); + $this->assertCount(1, $indexes); $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); $this->assertContains('idx_ttl_expires', $indexIds); - $this->assertContains('idx_ttl_deleted', $indexIds); + $this->assertNotContains('idx_ttl_deleted', $indexIds); try { $database->createIndex( @@ -2147,10 +2151,10 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void [Database::ORDER_ASC], 172800 // 48 hours ); - $this->fail('Expected exception for duplicate TTL index on same attribute'); + $this->fail('Expected exception for creating a second TTL index in a collection'); } catch (Exception $e) { $this->assertInstanceOf(DatabaseException::class, $e); - $this->assertStringContainsString('There is already an index with the same attributes and orders', $e->getMessage()); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); } $this->assertTrue($database->deleteIndex($col, 'idx_ttl_expires')); @@ -2158,9 +2162,9 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void $this->assertTrue( $database->createIndex( $col, - 'idx_ttl_expires_new', + 'idx_ttl_deleted', Database::INDEX_TTL, - ['expiresAt'], + ['deletedAt'], [], [Database::ORDER_ASC], 1800 // 30 minutes @@ -2169,11 +2173,10 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void $collection = $database->getCollection($col); $indexes = $collection->getAttribute('indexes'); - $this->assertCount(2, $indexes); + $this->assertCount(1, $indexes); $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); $this->assertNotContains('idx_ttl_expires', $indexIds); - $this->assertContains('idx_ttl_expires_new', $indexIds); $this->assertContains('idx_ttl_deleted', $indexIds); $col3 = uniqid('sl_ttl_dup_collection'); @@ -2212,7 +2215,7 @@ public function testSchemalessTTLIndexDuplicatePrevention(): void $this->fail('Expected exception for duplicate TTL indexes in createCollection'); } catch (Exception $e) { $this->assertInstanceOf(DatabaseException::class, $e); - $this->assertStringContainsString('Index already exists', $e->getMessage()); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); } $database->deleteCollection($col); diff --git a/tests/unit/Validator/IndexTest.php b/tests/unit/Validator/IndexTest.php index 1722c7301..c1e55f8b6 100644 --- a/tests/unit/Validator/IndexTest.php +++ b/tests/unit/Validator/IndexTest.php @@ -724,7 +724,7 @@ public function testTTLIndexValidation(): void ]); $this->assertTrue($validator->isValid($validIndexMin)); - // Invalid: TTL index on same attribute when another TTL index already exists + // Invalid: any additional TTL index when another TTL index already exists $collection->setAttribute('indexes', $validIndex, Document::SET_TYPE_APPEND); $validatorWithExisting = new Index( $collection->getAttribute('attributes'), @@ -756,7 +756,7 @@ public function testTTLIndexValidation(): void 'ttl' => 7200, ]); $this->assertFalse($validatorWithExisting->isValid($duplicateTTLIndex)); - $this->assertStringContainsString('There is already an index with the same attributes and orders', $validatorWithExisting->getDescription()); + $this->assertEquals('There can be only one TTL index in a collection', $validatorWithExisting->getDescription()); // Validator with supportForTrigramIndexes disabled should reject TTL $validatorNoSupport = new Index($collection->getAttribute('attributes'), $collection->getAttribute('indexes', []), 768, [], false, false, false, false, false, false, false, false, false); From cc7fca31a55a8921fd0d39e224198bc4e8e1d7de Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 14 Jan 2026 13:10:00 +0530 Subject: [PATCH 15/16] removed empty orders from the ttl index validator --- src/Database/Validator/Index.php | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php index 4261b9851..fda8543f8 100644 --- a/src/Database/Validator/Index.php +++ b/src/Database/Validator/Index.php @@ -757,11 +757,6 @@ public function checkTTLIndexes(Document $index): bool return false; } - if (empty($orders)) { - $this->message = 'TTL index need explicit orders. Add the orders to create this index.'; - return false; - } - $attributeName = $attributes[0] ?? ''; $attribute = $this->attributes[\strtolower($attributeName)] ?? new Document(); $attributeType = $attribute->getAttribute('type', ''); From 3b388c8e41afc44b6b198fcbf8e198a3e31000c8 Mon Sep 17 00:00:00 2001 From: ArnabChatterjee20k Date: Wed, 14 Jan 2026 14:09:08 +0530 Subject: [PATCH 16/16] changed ttl check --- src/Database/Validator/Index.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php index fda8543f8..4d32806db 100644 --- a/src/Database/Validator/Index.php +++ b/src/Database/Validator/Index.php @@ -766,7 +766,7 @@ public function checkTTLIndexes(Document $index): bool return false; } - if ($ttl <= 0) { + if ($ttl < 1) { $this->message = 'TTL must be at least 1 second'; return false; }