diff --git a/src/Database/Adapter.php b/src/Database/Adapter.php index 49a33e403..3b010ad89 100644 --- a/src/Database/Adapter.php +++ b/src/Database/Adapter.php @@ -666,10 +666,12 @@ abstract public function renameIndex(string $collection, string $old, string $ne * @param array $lengths * @param array $orders * @param array $indexAttributeTypes + * @param array $collation + * @param int $ttl * * @return bool */ - abstract public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool; + abstract public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool; /** * Delete Index @@ -1483,4 +1485,14 @@ public function getSupportForRegex(): bool { return $this->getSupportForPCRERegex() || $this->getSupportForPOSIXRegex(); } + + /** + * Are ttl indexes supported? + * + * @return bool + */ + public function getSupportForTTLIndexes(): bool + { + return false; + } } diff --git a/src/Database/Adapter/MariaDB.php b/src/Database/Adapter/MariaDB.php index b8110b039..6ae88336f 100644 --- a/src/Database/Adapter/MariaDB.php +++ b/src/Database/Adapter/MariaDB.php @@ -715,7 +715,7 @@ public function renameIndex(string $collection, string $old, string $new): bool * @return bool * @throws DatabaseException */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool { $metadataCollection = new Document(['$id' => Database::METADATA]); $collection = $this->getDocument($metadataCollection, $collection); @@ -2255,4 +2255,9 @@ public function getSupportForPOSIXRegex(): bool { return false; } + + public function getSupportForTTLIndexes(): bool + { + return false; + } } diff --git a/src/Database/Adapter/Mongo.php b/src/Database/Adapter/Mongo.php index fdcdd80c2..0b82a9e5a 100644 --- a/src/Database/Adapter/Mongo.php +++ b/src/Database/Adapter/Mongo.php @@ -488,7 +488,7 @@ public function createCollection(string $name, array $attributes = [], array $in $orders = $index->getAttribute('orders'); // If sharedTables, always add _tenant as the first key - if ($this->sharedTables) { + if ($this->shouldAddTenantToIndex($index)) { $key['_tenant'] = $this->getOrder(Database::ORDER_ASC); } @@ -508,6 +508,9 @@ public function createCollection(string $name, array $attributes = [], array $in $order = $this->getOrder($this->filter($orders[$j] ?? Database::ORDER_ASC)); $unique = true; break; + case Database::INDEX_TTL: + $order = $this->getOrder($this->filter($orders[$j] ?? Database::ORDER_ASC)); + break; default: // index not supported return false; @@ -526,6 +529,14 @@ public function createCollection(string $name, array $attributes = [], array $in $newIndexes[$i]['default_language'] = 'none'; } + // Handle TTL indexes + if ($index->getAttribute('type') === Database::INDEX_TTL) { + $ttl = $index->getAttribute('ttl', 0); + if ($ttl > 0) { + $newIndexes[$i]['expireAfterSeconds'] = $ttl; + } + } + // Add partial filter for indexes to avoid indexing null values if (in_array($index->getAttribute('type'), [ Database::INDEX_UNIQUE, @@ -901,10 +912,11 @@ public function deleteRelationship( * @param array $orders * @param array $indexAttributeTypes * @param array $collation + * @param int $ttl * @return bool * @throws Exception */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool { $name = $this->getNamespace() . '_' . $this->filter($collection); $id = $this->filter($id); @@ -913,7 +925,7 @@ public function createIndex(string $collection, string $id, string $type, array $indexes['name'] = $id; // If sharedTables, always add _tenant as the first key - if ($this->sharedTables) { + if ($this->shouldAddTenantToIndex($type)) { $indexes['key']['_tenant'] = $this->getOrder(Database::ORDER_ASC); } @@ -933,6 +945,8 @@ public function createIndex(string $collection, string $id, string $type, array case Database::INDEX_UNIQUE: $indexes['unique'] = true; break; + case Database::INDEX_TTL: + break; default: return false; } @@ -961,6 +975,11 @@ public function createIndex(string $collection, string $id, string $type, array $indexes['default_language'] = 'none'; } + // Handle TTL indexes + if ($type === Database::INDEX_TTL && $ttl > 0) { + $indexes['expireAfterSeconds'] = $ttl; + } + // Add partial filter for indexes to avoid indexing null values if (in_array($type, [Database::INDEX_UNIQUE, Database::INDEX_KEY])) { $partialFilter = []; @@ -1073,7 +1092,7 @@ public function renameIndex(string $collection, string $old, string $new): bool try { $deletedindex = $this->deleteIndex($collection, $old); - $createdindex = $this->createIndex($collection, $new, $index['type'], $index['attributes'], $index['lengths'] ?? [], $index['orders'] ?? [], $indexAttributeTypes); + $createdindex = $this->createIndex($collection, $new, $index['type'], $index['attributes'], $index['lengths'] ?? [], $index['orders'] ?? [], $indexAttributeTypes, [], $index['ttl'] ?? 0); } catch (\Exception $e) { throw $this->processException($e); } @@ -1235,30 +1254,7 @@ public function castingAfter(Document $collection, Document $document): Document $node = (int)$node; break; case Database::VAR_DATETIME: - if ($node instanceof UTCDateTime) { - // Handle UTCDateTime objects - $node = DateTime::format($node->toDateTime()); - } elseif (is_array($node) && isset($node['$date'])) { - // Handle Extended JSON format from (array) cast - // Format: {"$date":{"$numberLong":"1760405478290"}} - if (is_array($node['$date']) && isset($node['$date']['$numberLong'])) { - $milliseconds = (int)$node['$date']['$numberLong']; - $seconds = intdiv($milliseconds, 1000); - $microseconds = ($milliseconds % 1000) * 1000; - $dateTime = \DateTime::createFromFormat('U.u', $seconds . '.' . str_pad((string)$microseconds, 6, '0')); - if ($dateTime) { - $dateTime->setTimezone(new \DateTimeZone('UTC')); - $node = DateTime::format($dateTime); - } - } - } elseif (is_string($node)) { - // Already a string, validate and pass through - try { - new \DateTime($node); - } catch (\Exception $e) { - // Invalid date string, skip - } - } + $node = $this->convertUTCDateToString($node); break; case Database::VAR_OBJECT: // Convert stdClass objects to arrays for object attributes @@ -1279,6 +1275,8 @@ public function castingAfter(Document $collection, Document $document): Document // mongodb results out a stdclass for objects if (is_object($value) && get_class($value) === stdClass::class) { $document->setAttribute($key, $this->convertStdClassToArray($value)); + } elseif ($value instanceof UTCDateTime) { + $document->setAttribute($key, $this->convertUTCDateToString($value)); } } } @@ -1361,6 +1359,24 @@ public function castingBefore(Document $collection, Document $document): Documen unset($node); $document->setAttribute($key, ($array) ? $value : $value[0]); } + $indexes = $collection->getAttribute('indexes'); + $ttlIndexes = array_filter($indexes, fn ($index) => $index->getAttribute('type') === Database::INDEX_TTL); + + if (!$this->getSupportForAttributes()) { + foreach ($document->getArrayCopy() as $key => $value) { + if (in_array($this->getInternalKeyForAttribute($key), Database::INTERNAL_ATTRIBUTE_KEYS)) { + continue; + } + if (is_string($value) && (in_array($key, $ttlIndexes) || $this->isExtendedISODatetime($value))) { + try { + $newValue = new UTCDateTime(new \DateTime($value)); + $document->setAttribute($key, $newValue); + } catch (\Throwable $th) { + // skip -> a valid string + } + } + } + } return $document; } @@ -2652,6 +2668,25 @@ protected function getOrder(string $order): int }; } + /** + * Check if tenant should be added to index + * + * @param Document|string $indexOrType Index document or index type string + * @return bool + */ + protected function shouldAddTenantToIndex(Document|string $indexOrType): bool + { + if (!$this->sharedTables) { + return false; + } + + $indexType = $indexOrType instanceof Document + ? $indexOrType->getAttribute('type') + : $indexOrType; + + return $indexType !== Database::INDEX_TTL; + } + /** * @param array $selections * @param string $prefix @@ -3395,4 +3430,128 @@ public function getSupportForTrigramIndex(): bool { return false; } + + public function getSupportForTTLIndexes(): bool + { + return true; + } + + protected function isExtendedISODatetime(string $val): bool + { + /** + * Min: + * YYYY-MM-DDTHH:mm:ssZ (20) + * YYYY-MM-DDTHH:mm:ss+HH:MM (25) + * + * Max: + * YYYY-MM-DDTHH:mm:ss.fffffZ (26) + * YYYY-MM-DDTHH:mm:ss.fffff+HH:MM (31) + */ + + $len = strlen($val); + + // absolute minimum + if ($len < 20) { + return false; + } + + // fixed datetime fingerprints + if ( + !isset($val[19]) || + $val[4] !== '-' || + $val[7] !== '-' || + $val[10] !== 'T' || + $val[13] !== ':' || + $val[16] !== ':' + ) { + return false; + } + + // timezone detection + $hasZ = ($val[$len - 1] === 'Z'); + + $hasOffset = ( + $len >= 25 && + ($val[$len - 6] === '+' || $val[$len - 6] === '-') && + $val[$len - 3] === ':' + ); + + if (!$hasZ && !$hasOffset) { + return false; + } + + if ($hasOffset && $len > 31) { + return false; + } + + if ($hasZ && $len > 26) { + return false; + } + + $digitPositions = [ + 0,1,2,3, + 5,6, + 8,9, + 11,12, + 14,15, + 17,18 + ]; + + $timeEnd = $hasZ ? $len - 1 : $len - 6; + + // fractional seconds + if ($timeEnd > 19) { + if ($val[19] !== '.' || $timeEnd < 21) { + return false; + } + for ($i = 20; $i < $timeEnd; $i++) { + $digitPositions[] = $i; + } + } + + // timezone offset numeric digits + if ($hasOffset) { + foreach ([$len - 5, $len - 4, $len - 2, $len - 1] as $i) { + $digitPositions[] = $i; + } + } + + foreach ($digitPositions as $i) { + if (!ctype_digit($val[$i])) { + return false; + } + } + + return true; + } + + protected function convertUTCDateToString(mixed $node): mixed + { + if ($node instanceof UTCDateTime) { + // Handle UTCDateTime objects + $node = DateTime::format($node->toDateTime()); + } elseif (is_array($node) && isset($node['$date'])) { + // Handle Extended JSON format from (array) cast + // Format: {"$date":{"$numberLong":"1760405478290"}} + if (is_array($node['$date']) && isset($node['$date']['$numberLong'])) { + $milliseconds = (int)$node['$date']['$numberLong']; + $seconds = intdiv($milliseconds, 1000); + $microseconds = ($milliseconds % 1000) * 1000; + $dateTime = \DateTime::createFromFormat('U.u', $seconds . '.' . str_pad((string)$microseconds, 6, '0')); + if ($dateTime) { + $dateTime->setTimezone(new \DateTimeZone('UTC')); + $node = DateTime::format($dateTime); + } + } + } elseif (is_string($node)) { + // Already a string, validate and pass through + try { + new \DateTime($node); + } catch (\Exception $e) { + // Invalid date string, skip + } + } + + return $node; + } } diff --git a/src/Database/Adapter/MySQL.php b/src/Database/Adapter/MySQL.php index 308013738..a0294902a 100644 --- a/src/Database/Adapter/MySQL.php +++ b/src/Database/Adapter/MySQL.php @@ -317,4 +317,9 @@ protected function getOperatorSQL(string $column, \Utopia\Database\Operator $ope // For all other operators, use parent implementation return parent::getOperatorSQL($column, $operator, $bindIndex); } + + public function getSupportForTTLIndexes(): bool + { + return false; + } } diff --git a/src/Database/Adapter/Pool.php b/src/Database/Adapter/Pool.php index d70a836ea..9a42ce3cc 100644 --- a/src/Database/Adapter/Pool.php +++ b/src/Database/Adapter/Pool.php @@ -210,7 +210,7 @@ public function renameIndex(string $collection, string $old, string $new): bool return $this->delegate(__FUNCTION__, \func_get_args()); } - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool { return $this->delegate(__FUNCTION__, \func_get_args()); } @@ -649,4 +649,9 @@ public function getSupportForAlterLocks(): bool { return $this->delegate(__FUNCTION__, \func_get_args()); } + + public function getSupportForTTLIndexes(): bool + { + return $this->delegate(__FUNCTION__, \func_get_args()); + } } diff --git a/src/Database/Adapter/Postgres.php b/src/Database/Adapter/Postgres.php index 050180a0a..a6f701b0d 100644 --- a/src/Database/Adapter/Postgres.php +++ b/src/Database/Adapter/Postgres.php @@ -319,6 +319,7 @@ public function createCollection(string $name, array $attributes = [], array $in } } $indexOrders = $index->getAttribute('orders', []); + $indexTtl = $index->getAttribute('ttl', 0); if ($indexType === Database::INDEX_SPATIAL && count($indexOrders)) { throw new DatabaseException('Spatial indexes with explicit orders are not supported. Remove the orders to create this index.'); } @@ -329,7 +330,9 @@ public function createCollection(string $name, array $attributes = [], array $in $indexAttributes, [], $indexOrders, - $indexAttributesWithType + $indexAttributesWithType, + [], + $indexTtl ); } } catch (PDOException $e) { @@ -876,7 +879,7 @@ public function deleteRelationship( * @return bool */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool { $collection = $this->filter($collection); $id = $this->filter($id); @@ -2829,4 +2832,9 @@ protected function getSQLTable(string $name): string return "{$this->quote($this->getDatabase())}.{$this->quote($table)}"; } + + public function getSupportForTTLIndexes(): bool + { + return false; + } } diff --git a/src/Database/Adapter/SQLite.php b/src/Database/Adapter/SQLite.php index 948070654..0e9d1ba6f 100644 --- a/src/Database/Adapter/SQLite.php +++ b/src/Database/Adapter/SQLite.php @@ -216,8 +216,9 @@ public function createCollection(string $name, array $attributes = [], array $in $indexAttributes = $index->getAttribute('attributes', []); $indexLengths = $index->getAttribute('lengths', []); $indexOrders = $index->getAttribute('orders', []); + $indexTtl = $index->getAttribute('ttl', 0); - $this->createIndex($id, $indexId, $indexType, $indexAttributes, $indexLengths, $indexOrders); + $this->createIndex($id, $indexId, $indexType, $indexAttributes, $indexLengths, $indexOrders, [], [], $indexTtl); } $this->createIndex("{$id}_perms", '_index_1', Database::INDEX_UNIQUE, ['_document', '_type', '_permission'], [], []); @@ -455,7 +456,7 @@ public function renameIndex(string $collection, string $old, string $new): bool * @throws Exception * @throws PDOException */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool { $name = $this->filter($collection); $id = $this->filter($id); @@ -1908,4 +1909,9 @@ public function getSupportForPOSIXRegex(): bool { return false; } + + public function getSupportForTTLIndexes(): bool + { + return false; + } } diff --git a/src/Database/Database.php b/src/Database/Database.php index a2bc2da55..b19cd66c6 100644 --- a/src/Database/Database.php +++ b/src/Database/Database.php @@ -86,6 +86,7 @@ class Database public const INDEX_HNSW_COSINE = 'hnsw_cosine'; public const INDEX_HNSW_DOT = 'hnsw_dot'; public const INDEX_TRIGRAM = 'trigram'; + public const INDEX_TTL = 'ttl'; // Max limits public const MAX_INT = 2147483647; @@ -1584,6 +1585,14 @@ public function createCollection(string $id, array $attributes = [], array $inde throw new DuplicateException('Collection ' . $id . ' already exists'); } + // Enforce single TTL index per collection + if ($this->validate && $this->getAdapter()->getSupportForTTLIndexes()) { + $ttlIndexes = array_filter($indexes, fn (Document $idx) => $idx->getAttribute('type') === self::INDEX_TTL); + if (count($ttlIndexes) > 1) { + throw new IndexException('There can be only one TTL index in a collection'); + } + } + /** * Fix metadata index length & orders */ @@ -1648,6 +1657,7 @@ public function createCollection(string $id, array $attributes = [], array $inde $this->adapter->getSupportForIndex(), $this->adapter->getSupportForUniqueIndex(), $this->adapter->getSupportForFulltextIndex(), + $this->adapter->getSupportForTTLIndexes() ); foreach ($indexes as $index) { if (!$validator->isValid($index)) { @@ -2798,6 +2808,7 @@ public function updateAttribute(string $collection, string $id, ?string $type = $this->adapter->getSupportForIndex(), $this->adapter->getSupportForUniqueIndex(), $this->adapter->getSupportForFulltextIndex(), + $this->adapter->getSupportForTTLIndexes() ); foreach ($indexes as $index) { @@ -3603,6 +3614,7 @@ public function renameIndex(string $collection, string $old, string $new): bool * @param array $attributes * @param array $lengths * @param array $orders + * @param int $ttl * * @return bool * @throws AuthorizationException @@ -3613,14 +3625,13 @@ public function renameIndex(string $collection, string $old, string $new): bool * @throws StructureException * @throws Exception */ - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = [], int $ttl = 1): bool { if (empty($attributes)) { throw new DatabaseException('Missing attributes'); } $collection = $this->silent(fn () => $this->getCollection($collection)); - // index IDs are case-insensitive $indexes = $collection->getAttribute('indexes', []); @@ -3671,6 +3682,7 @@ public function createIndex(string $collection, string $id, string $type, array 'attributes' => $attributes, 'lengths' => $lengths, 'orders' => $orders, + 'ttl' => $ttl ]); if ($this->validate) { @@ -3693,6 +3705,7 @@ public function createIndex(string $collection, string $id, string $type, array $this->adapter->getSupportForIndex(), $this->adapter->getSupportForUniqueIndex(), $this->adapter->getSupportForFulltextIndex(), + $this->adapter->getSupportForTTLIndexes() ); if (!$validator->isValid($index)) { throw new IndexException($validator->getDescription()); @@ -3702,7 +3715,7 @@ public function createIndex(string $collection, string $id, string $type, array $collection->setAttribute('indexes', $index, Document::SET_TYPE_APPEND); try { - $created = $this->adapter->createIndex($collection->getId(), $id, $type, $attributes, $lengths, $orders, $indexAttributesWithTypes); + $created = $this->adapter->createIndex($collection->getId(), $id, $type, $attributes, $lengths, $orders, $indexAttributesWithTypes, [], $ttl); if (!$created) { throw new DatabaseException('Failed to create index'); diff --git a/src/Database/Mirror.php b/src/Database/Mirror.php index d9a6d09df..5944e4aa5 100644 --- a/src/Database/Mirror.php +++ b/src/Database/Mirror.php @@ -469,9 +469,9 @@ public function deleteAttribute(string $collection, string $id): bool return $result; } - public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = []): bool + public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths = [], array $orders = [], int $ttl = 1): bool { - $result = $this->source->createIndex($collection, $id, $type, $attributes, $lengths, $orders); + $result = $this->source->createIndex($collection, $id, $type, $attributes, $lengths, $orders, $ttl); if ($this->destination === null) { return $result; @@ -502,7 +502,8 @@ public function createIndex(string $collection, string $id, string $type, array $document->getAttribute('type'), $document->getAttribute('attributes'), $document->getAttribute('lengths'), - $document->getAttribute('orders') + $document->getAttribute('orders'), + $document->getAttribute('ttl', 0) ); } catch (\Throwable $err) { $this->logError('createIndex', $err); diff --git a/src/Database/Validator/Index.php b/src/Database/Validator/Index.php index e2fc70a0b..4d32806db 100644 --- a/src/Database/Validator/Index.php +++ b/src/Database/Validator/Index.php @@ -54,6 +54,7 @@ public function __construct( protected bool $supportForKeyIndexes = true, protected bool $supportForUniqueIndexes = true, protected bool $supportForFulltextIndexes = true, + protected bool $supportForTTLIndexes = false, ) { foreach ($attributes as $attribute) { $key = \strtolower($attribute->getAttribute('key', $attribute->getAttribute('$id'))); @@ -156,6 +157,9 @@ public function isValid($value): bool if (!$this->checkKeyUniqueFulltextSupport($value)) { return false; } + if (!$this->checkTTLIndexes($value)) { + return false; + } return true; } @@ -222,8 +226,15 @@ public function checkValidIndex(Document $index): bool } break; + case Database::INDEX_TTL: + if (!$this->supportForTTLIndexes) { + $this->message = 'TTL indexes are not supported'; + return false; + } + break; + default: - $this->message = 'Unknown index type: ' . $type . '. Must be one of ' . Database::INDEX_KEY . ', ' . Database::INDEX_UNIQUE . ', ' . Database::INDEX_FULLTEXT . ', ' . Database::INDEX_SPATIAL . ', ' . Database::INDEX_OBJECT . ', ' . Database::INDEX_HNSW_EUCLIDEAN . ', ' . Database::INDEX_HNSW_COSINE . ', ' . Database::INDEX_HNSW_DOT . ', '.Database::INDEX_TRIGRAM; + $this->message = 'Unknown index type: ' . $type . '. Must be one of ' . Database::INDEX_KEY . ', ' . Database::INDEX_UNIQUE . ', ' . Database::INDEX_FULLTEXT . ', ' . Database::INDEX_SPATIAL . ', ' . Database::INDEX_OBJECT . ', ' . Database::INDEX_HNSW_EUCLIDEAN . ', ' . Database::INDEX_HNSW_COSINE . ', ' . Database::INDEX_HNSW_DOT . ', '.Database::INDEX_TRIGRAM . ', '.Database::INDEX_TTL; return false; } return true; @@ -729,4 +740,50 @@ public function checkObjectIndexes(Document $index): bool return true; } + + public function checkTTLIndexes(Document $index): bool + { + $type = $index->getAttribute('type'); + + $attributes = $index->getAttribute('attributes', []); + $orders = $index->getAttribute('orders', []); + $ttl = $index->getAttribute('ttl', 0); + if ($type !== Database::INDEX_TTL) { + return true; + } + + if (count($attributes) !== 1) { + $this->message = 'TTL indexes must be created on a single datetime attribute.'; + return false; + } + + $attributeName = $attributes[0] ?? ''; + $attribute = $this->attributes[\strtolower($attributeName)] ?? new Document(); + $attributeType = $attribute->getAttribute('type', ''); + + if ($this->supportForAttributes && $attributeType !== Database::VAR_DATETIME) { + $this->message = 'TTL index can only be created on datetime attributes. Attribute "' . $attributeName . '" is of type "' . $attributeType . '"'; + return false; + } + + if ($ttl < 1) { + $this->message = 'TTL must be at least 1 second'; + return false; + } + + // Check if there's already a TTL index in this collection + foreach ($this->indexes as $existingIndex) { + if ($existingIndex->getId() === $index->getId()) { + continue; + } + + // Check if existing index is also a TTL index + if ($existingIndex->getAttribute('type') === Database::INDEX_TTL) { + $this->message = 'There can be only one TTL index in a collection'; + return false; + } + } + + return true; + } } diff --git a/tests/e2e/Adapter/Scopes/IndexTests.php b/tests/e2e/Adapter/Scopes/IndexTests.php index e5eda16d0..5364a4e65 100644 --- a/tests/e2e/Adapter/Scopes/IndexTests.php +++ b/tests/e2e/Adapter/Scopes/IndexTests.php @@ -784,4 +784,267 @@ public function testTrigramIndexValidation(): void $database->deleteCollection($collectionId); } } + + public function testTTLIndexes(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if (!$database->getAdapter()->getSupportForTTLIndexes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl'); + $database->createCollection($col); + + $database->createAttribute($col, 'expiresAt', Database::VAR_DATETIME, 0, false); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_valid', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 3600 // 1 hour TTL + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(1, $indexes); + $ttlIndex = $indexes[0]; + $this->assertEquals('idx_ttl_valid', $ttlIndex->getId()); + $this->assertEquals(Database::INDEX_TTL, $ttlIndex->getAttribute('type')); + $this->assertEquals(3600, $ttlIndex->getAttribute('ttl')); + + $now = new \DateTime(); + $future1 = (clone $now)->modify('+2 hours'); + $future2 = (clone $now)->modify('+1 hour'); + $past = (clone $now)->modify('-1 hour'); + + $database->createDocuments($col, [ + new Document([ + '$id' => 'doc1', + '$permissions' => $permissions, + 'expiresAt' => $future1->format(\DateTime::ATOM), + ]), + new Document([ + '$id' => 'doc2', + '$permissions' => $permissions, + 'expiresAt' => $future2->format(\DateTime::ATOM), + ]), + new Document([ + '$id' => 'doc3', + '$permissions' => $permissions, + 'expiresAt' => $past->format(\DateTime::ATOM), + ]) + ]); + + $this->assertTrue($database->deleteIndex($col, 'idx_ttl_valid')); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_min', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 1 // Minimum TTL + ) + ); + + $col2 = uniqid('sl_ttl_collection'); + + $expiresAtAttr = new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]); + + $ttlIndexDoc = new Document([ + '$id' => ID::custom('idx_ttl_collection'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200 // 2 hours + ]); + + $database->createCollection($col2, [$expiresAtAttr], [$ttlIndexDoc]); + + $collection2 = $database->getCollection($col2); + $indexes2 = $collection2->getAttribute('indexes'); + $this->assertCount(1, $indexes2); + $ttlIndex2 = $indexes2[0]; + $this->assertEquals('idx_ttl_collection', $ttlIndex2->getId()); + $this->assertEquals(7200, $ttlIndex2->getAttribute('ttl')); + + $database->deleteCollection($col); + $database->deleteCollection($col2); + } + + public function testTTLIndexDuplicatePrevention(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if (!$database->getAdapter()->getSupportForTTLIndexes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl_dup'); + $database->createCollection($col); + + $database->createAttribute($col, 'expiresAt', Database::VAR_DATETIME, 0, false); + $database->createAttribute($col, 'deletedAt', Database::VAR_DATETIME, 0, false); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expires', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 3600 // 1 hour + ) + ); + + try { + $database->createIndex( + $col, + 'idx_ttl_expires_duplicate', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 7200 // 2 hours + ); + $this->fail('Expected exception for creating a second TTL index in a collection'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); + } + + try { + $database->createIndex( + $col, + 'idx_ttl_deleted', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 86400 // 24 hours + ); + $this->fail('Expected exception for creating a second TTL index in a collection'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); + } + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(1, $indexes); + + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); + $this->assertContains('idx_ttl_expires', $indexIds); + $this->assertNotContains('idx_ttl_deleted', $indexIds); + + try { + $database->createIndex( + $col, + 'idx_ttl_deleted_duplicate', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 172800 // 48 hours + ); + $this->fail('Expected exception for creating a second TTL index in a collection'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); + } + + $this->assertTrue($database->deleteIndex($col, 'idx_ttl_expires')); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_deleted', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 1800 // 30 minutes + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(1, $indexes); + + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); + $this->assertNotContains('idx_ttl_expires', $indexIds); + $this->assertContains('idx_ttl_deleted', $indexIds); + + $col3 = uniqid('sl_ttl_dup_collection'); + + $expiresAtAttr = new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]); + + $ttlIndex1 = new Document([ + '$id' => ID::custom('idx_ttl_1'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 3600 + ]); + + $ttlIndex2 = new Document([ + '$id' => ID::custom('idx_ttl_2'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200 + ]); + + try { + $database->createCollection($col3, [$expiresAtAttr], [$ttlIndex1, $ttlIndex2]); + $this->fail('Expected exception for duplicate TTL indexes in createCollection'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); + } + + // Cleanup + $database->deleteCollection($col); + } } diff --git a/tests/e2e/Adapter/Scopes/SchemalessTests.php b/tests/e2e/Adapter/Scopes/SchemalessTests.php index 856d08263..5fbb69e16 100644 --- a/tests/e2e/Adapter/Scopes/SchemalessTests.php +++ b/tests/e2e/Adapter/Scopes/SchemalessTests.php @@ -6,6 +6,7 @@ use Throwable; use Utopia\Database\Database; use Utopia\Database\Document; +use Utopia\Database\Exception as DatabaseException; use Utopia\Database\Exception\Authorization as AuthorizationException; use Utopia\Database\Exception\Duplicate as DuplicateException; use Utopia\Database\Exception\Limit as LimitException; @@ -1029,19 +1030,36 @@ public function testSchemalessDates(): void $this->assertEquals('d1', $doc1->getId()); $this->assertTrue(is_string($doc1->getAttribute('curDate'))); - $this->assertEquals($curDate1, $doc1->getAttribute('curDate')); + // MongoDB converts ISO 8601 to 'Y-m-d H:i:s.v' format, so compare by parsing + $curDate1Value = $doc1->getAttribute('curDate'); + $parsedCurDate1 = new \DateTime($curDate1Value); + $parsedExpectedCurDate1 = new \DateTime($curDate1); + $this->assertEquals($parsedExpectedCurDate1->getTimestamp(), $parsedCurDate1->getTimestamp()); $this->assertTrue(is_string($doc1->getAttribute('$createdAt'))); $this->assertTrue(is_string($doc1->getAttribute('$updatedAt'))); - $this->assertEquals($createdAt1, $doc1->getAttribute('$createdAt')); - $this->assertEquals($updatedAt1, $doc1->getAttribute('$updatedAt')); + // Internal attributes should preserve format better, but verify by parsing for MongoDB + $createdAt1Value = $doc1->getAttribute('$createdAt'); + $updatedAt1Value = $doc1->getAttribute('$updatedAt'); + $parsedCreatedAt1 = new \DateTime($createdAt1Value); + $parsedUpdatedAt1 = new \DateTime($updatedAt1Value); + $parsedExpectedCreatedAt1 = new \DateTime($createdAt1); + $parsedExpectedUpdatedAt1 = new \DateTime($updatedAt1); + $this->assertEquals($parsedExpectedCreatedAt1->getTimestamp(), $parsedCreatedAt1->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt1->getTimestamp(), $parsedUpdatedAt1->getTimestamp()); $fetched1 = $database->getDocument($col, 'd1'); - $this->assertEquals($curDate1, $fetched1->getAttribute('curDate')); - $this->assertTrue(is_string($fetched1->getAttribute('curDate'))); + $fetchedCurDate1 = $fetched1->getAttribute('curDate'); + $this->assertTrue(is_string($fetchedCurDate1)); + $parsedFetchedCurDate1 = new \DateTime($fetchedCurDate1); + $this->assertEquals($parsedExpectedCurDate1->getTimestamp(), $parsedFetchedCurDate1->getTimestamp()); $this->assertTrue(is_string($fetched1->getAttribute('$createdAt'))); $this->assertTrue(is_string($fetched1->getAttribute('$updatedAt'))); - $this->assertEquals($createdAt1, $fetched1->getAttribute('$createdAt')); - $this->assertEquals($updatedAt1, $fetched1->getAttribute('$updatedAt')); + $fetchedCreatedAt1 = $fetched1->getAttribute('$createdAt'); + $fetchedUpdatedAt1 = $fetched1->getAttribute('$updatedAt'); + $parsedFetchedCreatedAt1 = new \DateTime($fetchedCreatedAt1); + $parsedFetchedUpdatedAt1 = new \DateTime($fetchedUpdatedAt1); + $this->assertEquals($parsedExpectedCreatedAt1->getTimestamp(), $parsedFetchedCreatedAt1->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt1->getTimestamp(), $parsedFetchedUpdatedAt1->getTimestamp()); // createDocuments with preserved dates $createdAt2 = '2001-02-03T04:05:06.000+00:00'; @@ -1073,14 +1091,32 @@ public function testSchemalessDates(): void $this->assertEquals(2, $countCreated); $fetched2 = $database->getDocument($col, 'd2'); - $this->assertEquals($curDate2, $fetched2->getAttribute('curDate')); - $this->assertEquals($createdAt2, $fetched2->getAttribute('$createdAt')); - $this->assertEquals($updatedAt2, $fetched2->getAttribute('$updatedAt')); + $fetchedCurDate2 = $fetched2->getAttribute('curDate'); + $parsedCurDate2 = new \DateTime($fetchedCurDate2); + $parsedExpectedCurDate2 = new \DateTime($curDate2); + $this->assertEquals($parsedExpectedCurDate2->getTimestamp(), $parsedCurDate2->getTimestamp()); + $fetchedCreatedAt2 = $fetched2->getAttribute('$createdAt'); + $fetchedUpdatedAt2 = $fetched2->getAttribute('$updatedAt'); + $parsedCreatedAt2 = new \DateTime($fetchedCreatedAt2); + $parsedUpdatedAt2 = new \DateTime($fetchedUpdatedAt2); + $parsedExpectedCreatedAt2 = new \DateTime($createdAt2); + $parsedExpectedUpdatedAt2 = new \DateTime($updatedAt2); + $this->assertEquals($parsedExpectedCreatedAt2->getTimestamp(), $parsedCreatedAt2->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt2->getTimestamp(), $parsedUpdatedAt2->getTimestamp()); $fetched3 = $database->getDocument($col, 'd3'); - $this->assertEquals($curDate3, $fetched3->getAttribute('curDate')); - $this->assertEquals($createdAt3, $fetched3->getAttribute('$createdAt')); - $this->assertEquals($updatedAt3, $fetched3->getAttribute('$updatedAt')); + $fetchedCurDate3 = $fetched3->getAttribute('curDate'); + $parsedCurDate3 = new \DateTime($fetchedCurDate3); + $parsedExpectedCurDate3 = new \DateTime($curDate3); + $this->assertEquals($parsedExpectedCurDate3->getTimestamp(), $parsedCurDate3->getTimestamp()); + $fetchedCreatedAt3 = $fetched3->getAttribute('$createdAt'); + $fetchedUpdatedAt3 = $fetched3->getAttribute('$updatedAt'); + $parsedCreatedAt3 = new \DateTime($fetchedCreatedAt3); + $parsedUpdatedAt3 = new \DateTime($fetchedUpdatedAt3); + $parsedExpectedCreatedAt3 = new \DateTime($createdAt3); + $parsedExpectedUpdatedAt3 = new \DateTime($updatedAt3); + $this->assertEquals($parsedExpectedCreatedAt3->getTimestamp(), $parsedCreatedAt3->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt3->getTimestamp(), $parsedUpdatedAt3->getTimestamp()); // updateDocument with preserved $updatedAt and custom date field $newCurDate1 = '2000-02-01T00:00:00.000+00:00'; @@ -1091,11 +1127,21 @@ public function testSchemalessDates(): void '$updatedAt' => $newUpdatedAt1, ])); }); - $this->assertEquals($newCurDate1, $updated1->getAttribute('curDate')); - $this->assertEquals($newUpdatedAt1, $updated1->getAttribute('$updatedAt')); + $updatedCurDate1 = $updated1->getAttribute('curDate'); + $updatedUpdatedAt1 = $updated1->getAttribute('$updatedAt'); + $parsedUpdatedCurDate1 = new \DateTime($updatedCurDate1); + $parsedUpdatedUpdatedAt1 = new \DateTime($updatedUpdatedAt1); + $parsedExpectedNewCurDate1 = new \DateTime($newCurDate1); + $parsedExpectedNewUpdatedAt1 = new \DateTime($newUpdatedAt1); + $this->assertEquals($parsedExpectedNewCurDate1->getTimestamp(), $parsedUpdatedCurDate1->getTimestamp()); + $this->assertEquals($parsedExpectedNewUpdatedAt1->getTimestamp(), $parsedUpdatedUpdatedAt1->getTimestamp()); $refetched1 = $database->getDocument($col, 'd1'); - $this->assertEquals($newCurDate1, $refetched1->getAttribute('curDate')); - $this->assertEquals($newUpdatedAt1, $refetched1->getAttribute('$updatedAt')); + $refetchedCurDate1 = $refetched1->getAttribute('curDate'); + $refetchedUpdatedAt1 = $refetched1->getAttribute('$updatedAt'); + $parsedRefetchedCurDate1 = new \DateTime($refetchedCurDate1); + $parsedRefetchedUpdatedAt1 = new \DateTime($refetchedUpdatedAt1); + $this->assertEquals($parsedExpectedNewCurDate1->getTimestamp(), $parsedRefetchedCurDate1->getTimestamp()); + $this->assertEquals($parsedExpectedNewUpdatedAt1->getTimestamp(), $parsedRefetchedUpdatedAt1->getTimestamp()); // updateDocuments with preserved $updatedAt over a subset $bulkCurDate = '2001-01-01T00:00:00.000+00:00'; @@ -1113,10 +1159,20 @@ public function testSchemalessDates(): void $this->assertEquals(2, $updatedCount); $afterBulk2 = $database->getDocument($col, 'd2'); $afterBulk3 = $database->getDocument($col, 'd3'); - $this->assertEquals($bulkCurDate, $afterBulk2->getAttribute('curDate')); - $this->assertEquals($bulkUpdatedAt, $afterBulk2->getAttribute('$updatedAt')); - $this->assertEquals($bulkCurDate, $afterBulk3->getAttribute('curDate')); - $this->assertEquals($bulkUpdatedAt, $afterBulk3->getAttribute('$updatedAt')); + $bulkCurDate2 = $afterBulk2->getAttribute('curDate'); + $bulkUpdatedAt2 = $afterBulk2->getAttribute('$updatedAt'); + $bulkCurDate3 = $afterBulk3->getAttribute('curDate'); + $bulkUpdatedAt3 = $afterBulk3->getAttribute('$updatedAt'); + $parsedBulkCurDate2 = new \DateTime($bulkCurDate2); + $parsedBulkUpdatedAt2 = new \DateTime($bulkUpdatedAt2); + $parsedBulkCurDate3 = new \DateTime($bulkCurDate3); + $parsedBulkUpdatedAt3 = new \DateTime($bulkUpdatedAt3); + $parsedExpectedBulkCurDate = new \DateTime($bulkCurDate); + $parsedExpectedBulkUpdatedAt = new \DateTime($bulkUpdatedAt); + $this->assertEquals($parsedExpectedBulkCurDate->getTimestamp(), $parsedBulkCurDate2->getTimestamp()); + $this->assertEquals($parsedExpectedBulkUpdatedAt->getTimestamp(), $parsedBulkUpdatedAt2->getTimestamp()); + $this->assertEquals($parsedExpectedBulkCurDate->getTimestamp(), $parsedBulkCurDate3->getTimestamp()); + $this->assertEquals($parsedExpectedBulkUpdatedAt->getTimestamp(), $parsedBulkUpdatedAt3->getTimestamp()); // upsertDocument: create new then update existing with preserved dates $createdAt4 = '2003-03-03T03:03:03.000+00:00'; @@ -1132,9 +1188,18 @@ public function testSchemalessDates(): void ])); }); $this->assertEquals('d4', $up1->getId()); - $this->assertEquals($curDate4, $up1->getAttribute('curDate')); - $this->assertEquals($createdAt4, $up1->getAttribute('$createdAt')); - $this->assertEquals($updatedAt4, $up1->getAttribute('$updatedAt')); + $up1CurDate4 = $up1->getAttribute('curDate'); + $up1CreatedAt4 = $up1->getAttribute('$createdAt'); + $up1UpdatedAt4 = $up1->getAttribute('$updatedAt'); + $parsedUp1CurDate4 = new \DateTime($up1CurDate4); + $parsedUp1CreatedAt4 = new \DateTime($up1CreatedAt4); + $parsedUp1UpdatedAt4 = new \DateTime($up1UpdatedAt4); + $parsedExpectedCurDate4 = new \DateTime($curDate4); + $parsedExpectedCreatedAt4 = new \DateTime($createdAt4); + $parsedExpectedUpdatedAt4 = new \DateTime($updatedAt4); + $this->assertEquals($parsedExpectedCurDate4->getTimestamp(), $parsedUp1CurDate4->getTimestamp()); + $this->assertEquals($parsedExpectedCreatedAt4->getTimestamp(), $parsedUp1CreatedAt4->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt4->getTimestamp(), $parsedUp1UpdatedAt4->getTimestamp()); $updatedAt4b = '2003-03-06T06:06:06.000+00:00'; $curDate4b = '2003-03-07T07:07:07.000+00:00'; @@ -1145,11 +1210,21 @@ public function testSchemalessDates(): void '$updatedAt' => $updatedAt4b, ])); }); - $this->assertEquals($curDate4b, $up2->getAttribute('curDate')); - $this->assertEquals($updatedAt4b, $up2->getAttribute('$updatedAt')); + $up2CurDate4b = $up2->getAttribute('curDate'); + $up2UpdatedAt4b = $up2->getAttribute('$updatedAt'); + $parsedUp2CurDate4b = new \DateTime($up2CurDate4b); + $parsedUp2UpdatedAt4b = new \DateTime($up2UpdatedAt4b); + $parsedExpectedCurDate4b = new \DateTime($curDate4b); + $parsedExpectedUpdatedAt4b = new \DateTime($updatedAt4b); + $this->assertEquals($parsedExpectedCurDate4b->getTimestamp(), $parsedUp2CurDate4b->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt4b->getTimestamp(), $parsedUp2UpdatedAt4b->getTimestamp()); $refetched4 = $database->getDocument($col, 'd4'); - $this->assertEquals($curDate4b, $refetched4->getAttribute('curDate')); - $this->assertEquals($updatedAt4b, $refetched4->getAttribute('$updatedAt')); + $refetched4CurDate4b = $refetched4->getAttribute('curDate'); + $refetched4UpdatedAt4b = $refetched4->getAttribute('$updatedAt'); + $parsedRefetched4CurDate4b = new \DateTime($refetched4CurDate4b); + $parsedRefetched4UpdatedAt4b = new \DateTime($refetched4UpdatedAt4b); + $this->assertEquals($parsedExpectedCurDate4b->getTimestamp(), $parsedRefetched4CurDate4b->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt4b->getTimestamp(), $parsedRefetched4UpdatedAt4b->getTimestamp()); // upsertDocuments: mix create and update with preserved dates $createdAt5 = '2004-04-01T01:01:01.000+00:00'; @@ -1177,13 +1252,28 @@ public function testSchemalessDates(): void $this->assertEquals(2, $upCount); $fetched5 = $database->getDocument($col, 'd5'); - $this->assertEquals($curDate5, $fetched5->getAttribute('curDate')); - $this->assertEquals($createdAt5, $fetched5->getAttribute('$createdAt')); - $this->assertEquals($updatedAt5, $fetched5->getAttribute('$updatedAt')); + $fetched5CurDate5 = $fetched5->getAttribute('curDate'); + $fetched5CreatedAt5 = $fetched5->getAttribute('$createdAt'); + $fetched5UpdatedAt5 = $fetched5->getAttribute('$updatedAt'); + $parsedFetched5CurDate5 = new \DateTime($fetched5CurDate5); + $parsedFetched5CreatedAt5 = new \DateTime($fetched5CreatedAt5); + $parsedFetched5UpdatedAt5 = new \DateTime($fetched5UpdatedAt5); + $parsedExpectedCurDate5 = new \DateTime($curDate5); + $parsedExpectedCreatedAt5 = new \DateTime($createdAt5); + $parsedExpectedUpdatedAt5 = new \DateTime($updatedAt5); + $this->assertEquals($parsedExpectedCurDate5->getTimestamp(), $parsedFetched5CurDate5->getTimestamp()); + $this->assertEquals($parsedExpectedCreatedAt5->getTimestamp(), $parsedFetched5CreatedAt5->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt5->getTimestamp(), $parsedFetched5UpdatedAt5->getTimestamp()); $fetched2b = $database->getDocument($col, 'd2'); - $this->assertEquals($curDate2b, $fetched2b->getAttribute('curDate')); - $this->assertEquals($updatedAt2b, $fetched2b->getAttribute('$updatedAt')); + $fetched2bCurDate2b = $fetched2b->getAttribute('curDate'); + $fetched2bUpdatedAt2b = $fetched2b->getAttribute('$updatedAt'); + $parsedFetched2bCurDate2b = new \DateTime($fetched2bCurDate2b); + $parsedFetched2bUpdatedAt2b = new \DateTime($fetched2bUpdatedAt2b); + $parsedExpectedCurDate2b = new \DateTime($curDate2b); + $parsedExpectedUpdatedAt2b = new \DateTime($updatedAt2b); + $this->assertEquals($parsedExpectedCurDate2b->getTimestamp(), $parsedFetched2bCurDate2b->getTimestamp()); + $this->assertEquals($parsedExpectedUpdatedAt2b->getTimestamp(), $parsedFetched2bUpdatedAt2b->getTimestamp()); // increase/decrease should not affect date types; ensure they remain strings $afterInc = $database->increaseDocumentAttribute($col, 'd1', 'counter', 5); @@ -1865,4 +1955,830 @@ public function testSchemalessNestedObjectAttributeQueries(): void $database->deleteCollection($col); } + + public function testSchemalessTTLIndexes(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl'); + $database->createCollection($col); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_valid', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 3600 // 1 hour TTL + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(1, $indexes); + $ttlIndex = $indexes[0]; + $this->assertEquals('idx_ttl_valid', $ttlIndex->getId()); + $this->assertEquals(Database::INDEX_TTL, $ttlIndex->getAttribute('type')); + $this->assertEquals(3600, $ttlIndex->getAttribute('ttl')); + + $now = new \DateTime(); + $future1 = (clone $now)->modify('+2 hours'); + $future2 = (clone $now)->modify('+1 hour'); + $past = (clone $now)->modify('-1 hour'); + + $doc1 = $database->createDocument($col, new Document([ + '$id' => 'doc1', + '$permissions' => $permissions, + 'expiresAt' => $future1->format(\DateTime::ATOM), + 'data' => 'will expire in 2 hours' + ])); + + $doc2 = $database->createDocument($col, new Document([ + '$id' => 'doc2', + '$permissions' => $permissions, + 'expiresAt' => $future2->format(\DateTime::ATOM), + 'data' => 'will expire in 1 hour' + ])); + + $doc3 = $database->createDocument($col, new Document([ + '$id' => 'doc3', + '$permissions' => $permissions, + 'expiresAt' => $past->format(\DateTime::ATOM), + 'data' => 'already expired' + ])); + + // Verify documents were created + $this->assertEquals('doc1', $doc1->getId()); + $this->assertEquals('doc2', $doc2->getId()); + $this->assertEquals('doc3', $doc3->getId()); + + $this->assertTrue($database->deleteIndex($col, 'idx_ttl_valid')); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_min', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 1 // Minimum TTL + ) + ); + + $col2 = uniqid('sl_ttl_collection'); + + $expiresAtAttr = new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]); + + $ttlIndexDoc = new Document([ + '$id' => ID::custom('idx_ttl_collection'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200 // 2 hours + ]); + + $database->createCollection($col2, [$expiresAtAttr], [$ttlIndexDoc]); + + $collection2 = $database->getCollection($col2); + $indexes2 = $collection2->getAttribute('indexes'); + $this->assertCount(1, $indexes2); + $ttlIndex2 = $indexes2[0]; + $this->assertEquals('idx_ttl_collection', $ttlIndex2->getId()); + $this->assertEquals(7200, $ttlIndex2->getAttribute('ttl')); + + $database->deleteCollection($col); + $database->deleteCollection($col2); + } + + public function testSchemalessTTLIndexDuplicatePrevention(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl_dup'); + $database->createCollection($col); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expires', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 3600 // 1 hour + ) + ); + + try { + $database->createIndex( + $col, + 'idx_ttl_expires_duplicate', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 7200 // 2 hours + ); + $this->fail('Expected exception for creating a second TTL index in a collection'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); + } + + try { + $database->createIndex( + $col, + 'idx_ttl_deleted', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 86400 // 24 hours + ); + $this->fail('Expected exception for creating a second TTL index in a collection'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); + } + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(1, $indexes); + + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); + $this->assertContains('idx_ttl_expires', $indexIds); + $this->assertNotContains('idx_ttl_deleted', $indexIds); + + try { + $database->createIndex( + $col, + 'idx_ttl_deleted_duplicate', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 172800 // 48 hours + ); + $this->fail('Expected exception for creating a second TTL index in a collection'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); + } + + $this->assertTrue($database->deleteIndex($col, 'idx_ttl_expires')); + + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_deleted', + Database::INDEX_TTL, + ['deletedAt'], + [], + [Database::ORDER_ASC], + 1800 // 30 minutes + ) + ); + + $collection = $database->getCollection($col); + $indexes = $collection->getAttribute('indexes'); + $this->assertCount(1, $indexes); + + $indexIds = array_map(fn ($idx) => $idx->getId(), $indexes); + $this->assertNotContains('idx_ttl_expires', $indexIds); + $this->assertContains('idx_ttl_deleted', $indexIds); + + $col3 = uniqid('sl_ttl_dup_collection'); + + $expiresAtAttr = new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]); + + $ttlIndex1 = new Document([ + '$id' => ID::custom('idx_ttl_1'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 3600 + ]); + + $ttlIndex2 = new Document([ + '$id' => ID::custom('idx_ttl_2'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200 + ]); + + try { + $database->createCollection($col3, [$expiresAtAttr], [$ttlIndex1, $ttlIndex2]); + $this->fail('Expected exception for duplicate TTL indexes in createCollection'); + } catch (Exception $e) { + $this->assertInstanceOf(DatabaseException::class, $e); + $this->assertStringContainsString('There can be only one TTL index in a collection', $e->getMessage()); + } + + $database->deleteCollection($col); + } + + public function testSchemalessDatetimeCreationAndFetching(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_datetime'); + $database->createCollection($col); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + // Create documents with ISO 8601 datetime strings (20-40 chars) + // MongoDB converts ISO 8601 to UTCDateTime and returns in 'Y-m-d H:i:s.v' format + $datetime1 = '2024-01-15T10:30:00.000+00:00'; + $datetime2 = '2024-02-20T14:45:30.123Z'; + $datetime3 = '2024-03-25T08:15:45.000000+05:30'; + + $doc1 = $database->createDocument($col, new Document([ + '$id' => 'dt1', + '$permissions' => $permissions, + 'eventDate' => $datetime1, + 'name' => 'Event 1' + ])); + + $doc2 = $database->createDocument($col, new Document([ + '$id' => 'dt2', + '$permissions' => $permissions, + 'eventDate' => $datetime2, + 'name' => 'Event 2' + ])); + + $doc3 = $database->createDocument($col, new Document([ + '$id' => 'dt3', + '$permissions' => $permissions, + 'eventDate' => $datetime3, + 'name' => 'Event 3' + ])); + + // Verify creation - check that datetime is stored and returned as string + $this->assertEquals('dt1', $doc1->getId()); + $eventDate1 = $doc1->getAttribute('eventDate'); + $this->assertTrue(is_string($eventDate1)); + $this->assertGreaterThanOrEqual(20, strlen($eventDate1)); + $this->assertLessThanOrEqual(40, strlen($eventDate1)); + + // Fetch and verify - MongoDB returns datetime in 'Y-m-d H:i:s.v' format + $fetched1 = $database->getDocument($col, 'dt1'); + $fetchedEventDate1 = $fetched1->getAttribute('eventDate'); + $this->assertTrue(is_string($fetchedEventDate1)); + $this->assertEquals('Event 1', $fetched1->getAttribute('name')); + + // Verify datetime values are equivalent by parsing (MongoDB converts to UTC) + $parsedInput1 = new \DateTime($datetime1); + $parsedOutput1 = new \DateTime($fetchedEventDate1); + $this->assertEquals($parsedInput1->getTimestamp(), $parsedOutput1->getTimestamp()); + + $fetched2 = $database->getDocument($col, 'dt2'); + $fetchedEventDate2 = $fetched2->getAttribute('eventDate'); + $this->assertTrue(is_string($fetchedEventDate2)); + $parsedInput2 = new \DateTime($datetime2); + $parsedOutput2 = new \DateTime($fetchedEventDate2); + $this->assertEquals($parsedInput2->getTimestamp(), $parsedOutput2->getTimestamp()); + + $fetched3 = $database->getDocument($col, 'dt3'); + $fetchedEventDate3 = $fetched3->getAttribute('eventDate'); + $this->assertTrue(is_string($fetchedEventDate3)); + // Verify it's a valid datetime string (format may vary slightly) + $this->assertGreaterThanOrEqual(20, strlen($fetchedEventDate3)); + $this->assertLessThanOrEqual(40, strlen($fetchedEventDate3)); + $parsedInput3 = new \DateTime($datetime3); + $parsedOutput3 = new \DateTime($fetchedEventDate3); + // MongoDB converts to UTC, so timestamps should match + $this->assertEquals($parsedInput3->getTimestamp(), $parsedOutput3->getTimestamp()); + + // Find all datetime documents + $allDocs = $database->find($col); + $this->assertCount(3, $allDocs); + + // Verify all documents are present + $allIds = array_map(fn ($doc) => $doc->getId(), $allDocs); + $this->assertContains('dt1', $allIds); + $this->assertContains('dt2', $allIds); + $this->assertContains('dt3', $allIds); + + // Update datetime + $newDatetime = '2024-12-31T23:59:59.999+00:00'; + $updated = $database->updateDocument($col, 'dt1', new Document([ + 'eventDate' => $newDatetime + ])); + $updatedEventDate = $updated->getAttribute('eventDate'); + $this->assertTrue(is_string($updatedEventDate)); + $this->assertGreaterThanOrEqual(20, strlen($updatedEventDate)); + $this->assertLessThanOrEqual(40, strlen($updatedEventDate)); + + $refetched = $database->getDocument($col, 'dt1'); + $refetchedEventDate = $refetched->getAttribute('eventDate'); + $this->assertTrue(is_string($refetchedEventDate)); + $parsedNewInput = new \DateTime($newDatetime); + $parsedNewOutput = new \DateTime($refetchedEventDate); + $this->assertEquals($parsedNewInput->getTimestamp(), $parsedNewOutput->getTimestamp()); + + $database->deleteCollection($col); + } + + public function testSchemalessTTLExpiry(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + if (!$database->getAdapter()->getSupportForTTLIndexes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_ttl_expiry'); + $database->createCollection($col); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + // Create TTL index with 60 seconds expiry + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expiresAt', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 10 + ) + ); + + $now = new \DateTime(); + $expiredTime = (clone $now)->modify('-10 seconds'); // Already expired + $futureTime = (clone $now)->modify('+120 seconds'); // Will expire in 2 minutes + + // Create mixed documents: some with expiresAt, some without + $doc1 = $database->createDocument($col, new Document([ + '$id' => 'expired_doc', + '$permissions' => $permissions, + 'expiresAt' => $expiredTime->format(\DateTime::ATOM), + 'data' => 'This should expire', + 'type' => 'temporary' + ])); + + $doc2 = $database->createDocument($col, new Document([ + '$id' => 'future_doc', + '$permissions' => $permissions, + 'expiresAt' => $futureTime->format(\DateTime::ATOM), + 'data' => 'This should not expire yet', + 'type' => 'temporary' + ])); + + $doc3 = $database->createDocument($col, new Document([ + '$id' => 'permanent_doc', + '$permissions' => $permissions, + 'data' => 'This should never expire', + 'type' => 'permanent' + ])); + + $doc4 = $database->createDocument($col, new Document([ + '$id' => 'another_permanent', + '$permissions' => $permissions, + 'data' => 'This should also never expire', + 'type' => 'permanent' + ])); + + // Verify all documents were created + $this->assertEquals('expired_doc', $doc1->getId()); + $this->assertEquals('future_doc', $doc2->getId()); + $this->assertEquals('permanent_doc', $doc3->getId()); + $this->assertEquals('another_permanent', $doc4->getId()); + + // Initial count should be 4 + $initialDocs = $database->find($col); + $this->assertCount(4, $initialDocs); + + // Wait for TTL to expire with retry loop + // Note: MongoDB TTL cleanup runs every 60 seconds, so we need to retry + $maxRetries = 15; // 15 retries * 5 seconds = 75 seconds max + $retryDelay = 5; // Wait 5 seconds between retries + $expiredDocDeleted = false; + + for ($i = 0; $i < $maxRetries; $i++) { + sleep($retryDelay); + + // Fetch collection to trigger TTL cleanup check + $collection = $database->getCollection($col); + $this->assertNotNull($collection); + + // Check if expired document is gone + $remainingDocs = $database->find($col); + $remainingIds = array_map(fn ($doc) => $doc->getId(), $remainingDocs); + + if (!in_array('expired_doc', $remainingIds)) { + $expiredDocDeleted = true; + break; + } + } + + // Assert that expired document was deleted + $this->assertTrue($expiredDocDeleted, 'Expired document should have been deleted after TTL expiry'); + + // After expiry, expired document should be gone + // Documents without expiresAt should remain + $remainingDocs = $database->find($col); + $remainingIds = array_map(fn ($doc) => $doc->getId(), $remainingDocs); + + // The expired document should be deleted + $this->assertNotContains('expired_doc', $remainingIds); + + // Documents without expiresAt should still exist + $this->assertContains('permanent_doc', $remainingIds); + $this->assertContains('another_permanent', $remainingIds); + + // Future document might still exist (depending on timing) or might be deleted + // The key test is that permanent docs remain and expired doc is gone + $this->assertGreaterThanOrEqual(2, count($remainingDocs)); // At least 2 permanent docs + $this->assertLessThanOrEqual(3, count($remainingDocs)); // At most 3 (if future doc still exists) + + // Verify permanent documents are still accessible + $permanent1 = $database->getDocument($col, 'permanent_doc'); + $this->assertFalse($permanent1->isEmpty()); + $this->assertEquals('permanent', $permanent1->getAttribute('type')); + + $permanent2 = $database->getDocument($col, 'another_permanent'); + $this->assertFalse($permanent2->isEmpty()); + $this->assertEquals('permanent', $permanent2->getAttribute('type')); + + // Verify expired document is gone + $expired = $database->getDocument($col, 'expired_doc'); + $this->assertTrue($expired->isEmpty()); + + $database->deleteCollection($col); + } + + public function testStringAndDatetime(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_str_datetime'); + $database->createCollection($col); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + // Create documents with mix of formatted dates (ISO 8601) and non-formatted dates (regular strings) + // Each document has two fields: str and datetime + $docs = [ + new Document([ + '$id' => 'doc1', + '$permissions' => $permissions, + 'str' => '2024-01-15T10:30:00.000+00:00', // ISO 8601 formatted date as string + 'datetime' => '2024-01-15T10:30:00.000+00:00' // ISO 8601 formatted date + ]), + new Document([ + '$id' => 'doc2', + '$permissions' => $permissions, + 'str' => 'just a regular string', // Non-formatted string + 'datetime' => '2024-02-20T14:45:30.123Z' // ISO 8601 formatted date + ]), + new Document([ + '$id' => 'doc3', + '$permissions' => $permissions, + 'str' => '2024-03-25T08:15:45.000000+05:30', // ISO 8601 formatted date as string + 'datetime' => 'not a date string' // Non-formatted string in datetime field + ]), + new Document([ + '$id' => 'doc4', + '$permissions' => $permissions, + 'str' => 'another string value', + 'datetime' => '2024-12-31T23:59:59.999+00:00' // ISO 8601 formatted date + ]), + new Document([ + '$id' => 'doc5', + '$permissions' => $permissions, + 'str' => '2024-06-15T12:00:00.000Z', // ISO 8601 formatted date as string + 'datetime' => '2024-06-15T12:00:00.000Z' // ISO 8601 formatted date + ]), + ]; + + $createdCount = $database->createDocuments($col, $docs); + $this->assertEquals(5, $createdCount); + + // Fetch all documents and validate + $allDocs = $database->find($col); + $this->assertCount(5, $allDocs); + + // Validate each document + $doc1 = $database->getDocument($col, 'doc1'); + $this->assertEquals('doc1', $doc1->getId()); + $this->assertTrue(is_string($doc1->getAttribute('str'))); + $this->assertTrue(is_string($doc1->getAttribute('datetime'))); + // str field should remain as string (even if it looks like a date) + $this->assertGreaterThanOrEqual(20, strlen($doc1->getAttribute('str'))); + $this->assertLessThanOrEqual(40, strlen($doc1->getAttribute('str'))); + // datetime field should be converted to MongoDB format if it's a valid ISO date + $datetime1 = $doc1->getAttribute('datetime'); + $this->assertTrue(is_string($datetime1)); + $this->assertGreaterThanOrEqual(20, strlen($datetime1)); + $this->assertLessThanOrEqual(40, strlen($datetime1)); + // Verify it's a valid datetime by parsing + $parsed1 = new \DateTime($datetime1); + $this->assertInstanceOf(\DateTime::class, $parsed1); + + $doc2 = $database->getDocument($col, 'doc2'); + $this->assertEquals('doc2', $doc2->getId()); + $this->assertEquals('just a regular string', $doc2->getAttribute('str')); + $datetime2 = $doc2->getAttribute('datetime'); + $this->assertTrue(is_string($datetime2)); + $parsed2 = new \DateTime($datetime2); + $this->assertInstanceOf(\DateTime::class, $parsed2); + + $doc3 = $database->getDocument($col, 'doc3'); + $this->assertEquals('doc3', $doc3->getId()); + $str3 = $doc3->getAttribute('str'); + $this->assertTrue(is_string($str3)); + $this->assertGreaterThanOrEqual(20, strlen($str3)); + $this->assertLessThanOrEqual(40, strlen($str3)); + // datetime field contains non-date string, should remain as string + $datetime3 = $doc3->getAttribute('datetime'); + $this->assertEquals('not a date string', $datetime3); + $this->assertTrue(is_string($datetime3)); + + $doc4 = $database->getDocument($col, 'doc4'); + $this->assertEquals('doc4', $doc4->getId()); + $this->assertEquals('another string value', $doc4->getAttribute('str')); + $datetime4 = $doc4->getAttribute('datetime'); + $this->assertTrue(is_string($datetime4)); + $parsed4 = new \DateTime($datetime4); + $this->assertInstanceOf(\DateTime::class, $parsed4); + + $doc5 = $database->getDocument($col, 'doc5'); + $this->assertEquals('doc5', $doc5->getId()); + $str5 = $doc5->getAttribute('str'); + $this->assertTrue(is_string($str5)); + $this->assertGreaterThanOrEqual(20, strlen($str5)); + $this->assertLessThanOrEqual(40, strlen($str5)); + $datetime5 = $doc5->getAttribute('datetime'); + $this->assertTrue(is_string($datetime5)); + $parsed5 = new \DateTime($datetime5); + $this->assertInstanceOf(\DateTime::class, $parsed5); + + // Verify all documents are present using simple find + $allDocs = $database->find($col); + $this->assertCount(5, $allDocs); + $allIds = array_map(fn ($doc) => $doc->getId(), $allDocs); + $this->assertContains('doc1', $allIds); + $this->assertContains('doc2', $allIds); + $this->assertContains('doc3', $allIds); + $this->assertContains('doc4', $allIds); + $this->assertContains('doc5', $allIds); + + $database->deleteCollection($col); + } + + public function testStringAndDateWithTTL(): void + { + /** @var Database $database */ + $database = static::getDatabase(); + + if ($database->getAdapter()->getSupportForAttributes()) { + $this->expectNotToPerformAssertions(); + return; + } + + if (!$database->getAdapter()->getSupportForTTLIndexes()) { + $this->expectNotToPerformAssertions(); + return; + } + + $col = uniqid('sl_str_date_ttl'); + $database->createCollection($col); + + $permissions = [ + Permission::read(Role::any()), + Permission::write(Role::any()), + Permission::update(Role::any()), + Permission::delete(Role::any()) + ]; + + // Create TTL index on expiresAt field + $this->assertTrue( + $database->createIndex( + $col, + 'idx_ttl_expiresAt', + Database::INDEX_TTL, + ['expiresAt'], + [], + [Database::ORDER_ASC], + 10 + ) + ); + + $now = new \DateTime(); + $expiredTime = (clone $now)->modify('-10 seconds'); // Already expired + $futureTime = (clone $now)->modify('+120 seconds'); // Will expire in 2 minutes + + // Create documents with mix of datetime values and random strings in expiresAt + $docs = [ + new Document([ + '$id' => 'doc_datetime_expired', + '$permissions' => $permissions, + 'expiresAt' => $expiredTime->format(\DateTime::ATOM), // Valid datetime - should expire + 'data' => 'This should expire', + 'type' => 'datetime' + ]), + new Document([ + '$id' => 'doc_datetime_future', + '$permissions' => $permissions, + 'expiresAt' => $futureTime->format(\DateTime::ATOM), // Valid datetime - future + 'data' => 'This should not expire yet', + 'type' => 'datetime' + ]), + new Document([ + '$id' => 'doc_string_random', + '$permissions' => $permissions, + 'expiresAt' => 'random_string_value_12345', // Random string - should not expire + 'data' => 'This should never expire', + 'type' => 'string' + ]), + new Document([ + '$id' => 'doc_string_another', + '$permissions' => $permissions, + 'expiresAt' => 'another_random_string_xyz', // Random string - should not expire + 'data' => 'This should also never expire', + 'type' => 'string' + ]), + new Document([ + '$id' => 'doc_datetime_valid', + '$permissions' => $permissions, + 'expiresAt' => $futureTime->format(\DateTime::ATOM), // Valid datetime - future + 'data' => 'This is a valid datetime', + 'type' => 'datetime' + ]), + ]; + + $createdCount = $database->createDocuments($col, $docs); + $this->assertEquals(5, $createdCount); + + // Verify all documents were created + $initialDocs = $database->find($col); + $this->assertCount(5, $initialDocs); + + // Verify documents with datetime values + $docDatetimeExpired = $database->getDocument($col, 'doc_datetime_expired'); + $this->assertFalse($docDatetimeExpired->isEmpty()); + $expiresAt1 = $docDatetimeExpired->getAttribute('expiresAt'); + $this->assertTrue(is_string($expiresAt1)); + // Should be converted to MongoDB datetime format if it was a valid ISO date + $this->assertGreaterThanOrEqual(20, strlen($expiresAt1)); + $this->assertLessThanOrEqual(40, strlen($expiresAt1)); + // Verify it can be parsed as datetime + $parsed1 = new \DateTime($expiresAt1); + $this->assertInstanceOf(\DateTime::class, $parsed1); + + $docDatetimeFuture = $database->getDocument($col, 'doc_datetime_future'); + $this->assertFalse($docDatetimeFuture->isEmpty()); + $expiresAt2 = $docDatetimeFuture->getAttribute('expiresAt'); + $this->assertTrue(is_string($expiresAt2)); + $parsed2 = new \DateTime($expiresAt2); + $this->assertInstanceOf(\DateTime::class, $parsed2); + + // Verify documents with random strings remain as strings + $docStringRandom = $database->getDocument($col, 'doc_string_random'); + $this->assertFalse($docStringRandom->isEmpty()); + $expiresAt3 = $docStringRandom->getAttribute('expiresAt'); + $this->assertEquals('random_string_value_12345', $expiresAt3); + $this->assertTrue(is_string($expiresAt3)); + // Should remain as the original string (not converted to datetime) + $this->assertEquals('random_string_value_12345', $expiresAt3); + + $docStringAnother = $database->getDocument($col, 'doc_string_another'); + $this->assertFalse($docStringAnother->isEmpty()); + $expiresAt4 = $docStringAnother->getAttribute('expiresAt'); + $this->assertEquals('another_random_string_xyz', $expiresAt4); + $this->assertTrue(is_string($expiresAt4)); + + $maxRetries = 25; + $retryDelay = 5; + $expiredDocDeleted = false; + + for ($i = 0; $i < $maxRetries; $i++) { + sleep($retryDelay); + + // Fetch collection to trigger TTL cleanup check + $collection = $database->getCollection($col); + $this->assertNotNull($collection); + + // Check if expired datetime document is gone + $remainingDocs = $database->find($col); + $remainingIds = array_map(fn ($doc) => $doc->getId(), $remainingDocs); + + if (!in_array('doc_datetime_expired', $remainingIds)) { + $expiredDocDeleted = true; + break; + } + } + + // Assert that expired document was deleted + $this->assertTrue($expiredDocDeleted, 'Expired datetime document should have been deleted after TTL expiry'); + + // After expiry, check remaining documents + $remainingDocs = $database->find($col); + $remainingIds = array_map(fn ($doc) => $doc->getId(), $remainingDocs); + + // The expired datetime document should be deleted + $this->assertNotContains('doc_datetime_expired', $remainingIds); + + // Documents with random strings should still exist (TTL doesn't affect non-datetime values) + $this->assertContains('doc_string_random', $remainingIds); + $this->assertContains('doc_string_another', $remainingIds); + + // Verify random string documents are still accessible with original string values + $remainingStringDoc1 = $database->getDocument($col, 'doc_string_random'); + $this->assertFalse($remainingStringDoc1->isEmpty()); + $this->assertEquals('random_string_value_12345', $remainingStringDoc1->getAttribute('expiresAt')); + $this->assertEquals('string', $remainingStringDoc1->getAttribute('type')); + + $remainingStringDoc2 = $database->getDocument($col, 'doc_string_another'); + $this->assertFalse($remainingStringDoc2->isEmpty()); + $this->assertEquals('another_random_string_xyz', $remainingStringDoc2->getAttribute('expiresAt')); + $this->assertEquals('string', $remainingStringDoc2->getAttribute('type')); + + // Future datetime documents might still exist or be deleted depending on timing + // But at minimum, we should have the string documents + $this->assertGreaterThanOrEqual(2, count($remainingDocs)); // At least 2 string docs + $this->assertLessThanOrEqual(4, count($remainingDocs)); // At most 4 (if future datetime docs still exist) + + $database->deleteCollection($col); + } } diff --git a/tests/unit/Validator/IndexTest.php b/tests/unit/Validator/IndexTest.php index 5dfe80e4e..c1e55f8b6 100644 --- a/tests/unit/Validator/IndexTest.php +++ b/tests/unit/Validator/IndexTest.php @@ -596,4 +596,171 @@ public function testTrigramIndexValidation(): void $this->assertFalse($validatorNoSupport->isValid($validIndex)); $this->assertEquals('Trigram indexes are not supported', $validatorNoSupport->getDescription()); } + + /** + * @throws Exception + */ + public function testTTLIndexValidation(): void + { + $collection = new Document([ + '$id' => ID::custom('test'), + 'name' => 'test', + 'attributes' => [ + new Document([ + '$id' => ID::custom('expiresAt'), + 'type' => Database::VAR_DATETIME, + 'format' => '', + 'size' => 0, + 'signed' => false, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => ['datetime'], + ]), + new Document([ + '$id' => ID::custom('name'), + 'type' => Database::VAR_STRING, + 'format' => '', + 'size' => 255, + 'signed' => true, + 'required' => false, + 'default' => null, + 'array' => false, + 'filters' => [], + ]), + ], + 'indexes' => [] + ]); + + // Validator with supportForTTLIndexes enabled + $validator = new Index( + $collection->getAttribute('attributes'), + $collection->getAttribute('indexes', []), + 768, + [], + false, // supportForArrayIndexes + false, // supportForSpatialIndexNull + false, // supportForSpatialIndexOrder + false, // supportForVectorIndexes + true, // supportForAttributes + true, // supportForMultipleFulltextIndexes + true, // supportForIdenticalIndexes + false, // supportForObjectIndexes + false, // supportForTrigramIndexes + false, // supportForSpatialIndexes + true, // supportForKeyIndexes + true, // supportForUniqueIndexes + true, // supportForFulltextIndexes + true // supportForTTLIndexes + ); + + // Valid: TTL index on single datetime attribute with valid TTL + $validIndex = new Document([ + '$id' => ID::custom('idx_ttl_valid'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 3600, + ]); + $this->assertTrue($validator->isValid($validIndex)); + + // Invalid: TTL index with ttl = 1 + $invalidIndexZero = new Document([ + '$id' => ID::custom('idx_ttl_zero'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 0, + ]); + $this->assertFalse($validator->isValid($invalidIndexZero)); + $this->assertEquals('TTL must be at least 1 second', $validator->getDescription()); + + // Invalid: TTL index with TTL < 0 + $invalidIndexNegative = new Document([ + '$id' => ID::custom('idx_ttl_negative'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => -100, + ]); + $this->assertFalse($validator->isValid($invalidIndexNegative)); + $this->assertEquals('TTL must be at least 1 second', $validator->getDescription()); + + // Invalid: TTL index on non-datetime attribute + $invalidIndexType = new Document([ + '$id' => ID::custom('idx_ttl_invalid_type'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['name'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 3600, + ]); + $this->assertFalse($validator->isValid($invalidIndexType)); + $this->assertStringContainsString('TTL index can only be created on datetime attributes', $validator->getDescription()); + + // Invalid: TTL index on multiple attributes + $invalidIndexMulti = new Document([ + '$id' => ID::custom('idx_ttl_multi'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt', 'name'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC, Database::ORDER_ASC], + 'ttl' => 3600, + ]); + $this->assertFalse($validator->isValid($invalidIndexMulti)); + $this->assertStringContainsString('TTL indexes must be created on a single datetime attribute', $validator->getDescription()); + + // Valid: TTL index with minimum valid TTL (1 second) + $validIndexMin = new Document([ + '$id' => ID::custom('idx_ttl_min'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 1, + ]); + $this->assertTrue($validator->isValid($validIndexMin)); + + // Invalid: any additional TTL index when another TTL index already exists + $collection->setAttribute('indexes', $validIndex, Document::SET_TYPE_APPEND); + $validatorWithExisting = new Index( + $collection->getAttribute('attributes'), + $collection->getAttribute('indexes', []), + 768, + [], + false, // supportForArrayIndexes + false, // supportForSpatialIndexNull + false, // supportForSpatialIndexOrder + false, // supportForVectorIndexes + true, // supportForAttributes + true, // supportForMultipleFulltextIndexes + true, // supportForIdenticalIndexes + false, // supportForObjectIndexes + false, // supportForTrigramIndexes + false, // supportForSpatialIndexes + true, // supportForKeyIndexes + true, // supportForUniqueIndexes + true, // supportForFulltextIndexes + true // supportForTTLIndexes + ); + + $duplicateTTLIndex = new Document([ + '$id' => ID::custom('idx_ttl_duplicate'), + 'type' => Database::INDEX_TTL, + 'attributes' => ['expiresAt'], + 'lengths' => [], + 'orders' => [Database::ORDER_ASC], + 'ttl' => 7200, + ]); + $this->assertFalse($validatorWithExisting->isValid($duplicateTTLIndex)); + $this->assertEquals('There can be only one TTL index in a collection', $validatorWithExisting->getDescription()); + + // Validator with supportForTrigramIndexes disabled should reject TTL + $validatorNoSupport = new Index($collection->getAttribute('attributes'), $collection->getAttribute('indexes', []), 768, [], false, false, false, false, false, false, false, false, false); + $this->assertFalse($validatorNoSupport->isValid($validIndex)); + $this->assertEquals('TTL indexes are not supported', $validatorNoSupport->getDescription()); + } }