Mongo ttl index support

src/Database/Adapter.php

@@ -666,10 +666,12 @@ abstract public function renameIndex(string $collection, string $old, string $ne
      * @param array<int> $lengths
      * @param array<string> $orders
      * @param array<string,string> $indexAttributeTypes
+     * @param array<string, mixed> $collation
+     * @param int $ttl
      *
      * @return bool
      */
-    abstract public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool;
+    abstract public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool;
 
     /**
      * Delete Index
@@ -1483,4 +1485,14 @@ public function getSupportForRegex(): bool
     {
         return $this->getSupportForPCRERegex() || $this->getSupportForPOSIXRegex();
     }
+
+    /**
+     * Are ttl indexes supported?
+     *
+     * @return bool
+     */
+    public function getSupportForTTLIndexes(): bool
+    {
+        return false;
+    }
 }

src/Database/Adapter/MariaDB.php

@@ -715,7 +715,7 @@ public function renameIndex(string $collection, string $old, string $new): bool
      * @return bool
      * @throws DatabaseException
      */
-    public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool
+    public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool
     {
         $metadataCollection = new Document(['$id' => Database::METADATA]);
         $collection = $this->getDocument($metadataCollection, $collection);
@@ -2255,4 +2255,9 @@ public function getSupportForPOSIXRegex(): bool
     {
         return false;
     }
+
+    public function getSupportForTTLIndexes(): bool
+    {
+        return false;
+    }
 }

src/Database/Adapter/Mongo.php

@@ -488,7 +488,7 @@ public function createCollection(string $name, array $attributes = [], array $in
                 $orders = $index->getAttribute('orders');
 
                 // If sharedTables, always add _tenant as the first key
-                if ($this->sharedTables) {
+                if ($this->shouldAddTenantToIndex($index)) {
                     $key['_tenant'] = $this->getOrder(Database::ORDER_ASC);
                 }
 
@@ -508,6 +508,9 @@ public function createCollection(string $name, array $attributes = [], array $in
                             $order = $this->getOrder($this->filter($orders[$j] ?? Database::ORDER_ASC));
                             $unique = true;
                             break;
+                        case Database::INDEX_TTL:
+                            $order = $this->getOrder($this->filter($orders[$j] ?? Database::ORDER_ASC));
+                            break;
                         default:
                             // index not supported
                             return false;
@@ -526,6 +529,14 @@ public function createCollection(string $name, array $attributes = [], array $in
                     $newIndexes[$i]['default_language'] = 'none';
                 }
 
+                // Handle TTL indexes
+                if ($index->getAttribute('type') === Database::INDEX_TTL) {
+                    $ttl = $index->getAttribute('ttl', 0);
+                    if ($ttl > 0) {
+                        $newIndexes[$i]['expireAfterSeconds'] = $ttl;
+                    }
+                }
+
                 // Add partial filter for indexes to avoid indexing null values
                 if (in_array($index->getAttribute('type'), [
                     Database::INDEX_UNIQUE,
@@ -901,10 +912,11 @@ public function deleteRelationship(
      * @param array<string> $orders
      * @param array<string, string> $indexAttributeTypes
      * @param array<string, mixed> $collation
+     * @param int $ttl
      * @return bool
      * @throws Exception
      */
-    public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = []): bool
+    public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool
     {
         $name = $this->getNamespace() . '_' . $this->filter($collection);
         $id = $this->filter($id);
@@ -913,7 +925,7 @@ public function createIndex(string $collection, string $id, string $type, array
         $indexes['name'] = $id;
 
         // If sharedTables, always add _tenant as the first key
-        if ($this->sharedTables) {
+        if ($this->shouldAddTenantToIndex($type)) {
             $indexes['key']['_tenant'] = $this->getOrder(Database::ORDER_ASC);
         }
 
@@ -933,6 +945,8 @@ public function createIndex(string $collection, string $id, string $type, array
                 case Database::INDEX_UNIQUE:
                     $indexes['unique'] = true;
                     break;
+                case Database::INDEX_TTL:
+                    break;
                 default:
                     return false;
             }
@@ -961,6 +975,11 @@ public function createIndex(string $collection, string $id, string $type, array
             $indexes['default_language'] = 'none';
         }
 
+        // Handle TTL indexes
+        if ($type === Database::INDEX_TTL && $ttl > 0) {
+            $indexes['expireAfterSeconds'] = $ttl;
+        }
+
         // Add partial filter for indexes to avoid indexing null values
         if (in_array($type, [Database::INDEX_UNIQUE, Database::INDEX_KEY])) {
             $partialFilter = [];
@@ -1073,7 +1092,7 @@ public function renameIndex(string $collection, string $old, string $new): bool
 
         try {
             $deletedindex = $this->deleteIndex($collection, $old);
-            $createdindex = $this->createIndex($collection, $new, $index['type'], $index['attributes'], $index['lengths'] ?? [], $index['orders'] ?? [], $indexAttributeTypes);
+            $createdindex = $this->createIndex($collection, $new, $index['type'], $index['attributes'], $index['lengths'] ?? [], $index['orders'] ?? [], $indexAttributeTypes, [], $index['ttl'] ?? 0);
         } catch (\Exception $e) {
             throw $this->processException($e);
         }
@@ -1235,30 +1254,7 @@ public function castingAfter(Document $collection, Document $document): Document
                         $node = (int)$node;
                         break;
                     case Database::VAR_DATETIME:
-                        if ($node instanceof UTCDateTime) {
-                            // Handle UTCDateTime objects
-                            $node = DateTime::format($node->toDateTime());
-                        } elseif (is_array($node) && isset($node['$date'])) {
-                            // Handle Extended JSON format from (array) cast
-                            // Format: {"$date":{"$numberLong":"1760405478290"}}
-                            if (is_array($node['$date']) && isset($node['$date']['$numberLong'])) {
-                                $milliseconds = (int)$node['$date']['$numberLong'];
-                                $seconds = intdiv($milliseconds, 1000);
-                                $microseconds = ($milliseconds % 1000) * 1000;
-                                $dateTime = \DateTime::createFromFormat('U.u', $seconds . '.' . str_pad((string)$microseconds, 6, '0'));
-                                if ($dateTime) {
-                                    $dateTime->setTimezone(new \DateTimeZone('UTC'));
-                                    $node = DateTime::format($dateTime);
-                                }
-                            }
-                        } elseif (is_string($node)) {
-                            // Already a string, validate and pass through
-                            try {
-                                new \DateTime($node);
-                            } catch (\Exception $e) {
-                                // Invalid date string, skip
-                            }
-                        }
+                        $node = $this->convertUTCDateToString($node);
                         break;
                     case Database::VAR_OBJECT:
                         // Convert stdClass objects to arrays for object attributes
@@ -1279,6 +1275,8 @@ public function castingAfter(Document $collection, Document $document): Document
                 // mongodb results out a stdclass for objects
                 if (is_object($value) && get_class($value) === stdClass::class) {
                     $document->setAttribute($key, $this->convertStdClassToArray($value));
+                } elseif ($value instanceof UTCDateTime) {
+                    $document->setAttribute($key, $this->convertUTCDateToString($value));
                 }
             }
         }
@@ -1361,6 +1359,24 @@ public function castingBefore(Document $collection, Document $document): Documen
             unset($node);
             $document->setAttribute($key, ($array) ? $value : $value[0]);
         }
+        $indexes = $collection->getAttribute('indexes');
+        $ttlIndexes = array_filter($indexes, fn ($index) => $index->getAttribute('type') === Database::INDEX_TTL);
+
+        if (!$this->getSupportForAttributes()) {
+            foreach ($document->getArrayCopy() as $key => $value) {
+                if (in_array($this->getInternalKeyForAttribute($key), Database::INTERNAL_ATTRIBUTE_KEYS)) {
+                    continue;
+                }
+                if (is_string($value) && (in_array($key, $ttlIndexes) || $this->isExtendedISODatetime($value))) {
+                    try {
+                        $newValue = new UTCDateTime(new \DateTime($value));
+                        $document->setAttribute($key, $newValue);
+                    } catch (\Throwable $th) {
+                        // skip -> a valid string
+                    }
+                }
+            }
+        }
 
         return $document;
     }
@@ -2652,6 +2668,25 @@ protected function getOrder(string $order): int
         };
     }
 
+    /**
+     * Check if tenant should be added to index
+     *
+     * @param Document|string $indexOrType Index document or index type string
+     * @return bool
+     */
+    protected function shouldAddTenantToIndex(Document|string $indexOrType): bool
+    {
+        if (!$this->sharedTables) {
+            return false;
+        }
+
+        $indexType = $indexOrType instanceof Document
+            ? $indexOrType->getAttribute('type')
+            : $indexOrType;
+
+        return $indexType !== Database::INDEX_TTL;
+    }
+
     /**
      * @param array<string> $selections
      * @param string $prefix
@@ -3395,4 +3430,128 @@ public function getSupportForTrigramIndex(): bool
     {
         return false;
     }
+
+    public function getSupportForTTLIndexes(): bool
+    {
+        return true;
+    }
+
+    protected function isExtendedISODatetime(string $val): bool
+    {
+        /**
+         * Min:
+         *   YYYY-MM-DDTHH:mm:ssZ             (20)
+         *   YYYY-MM-DDTHH:mm:ss+HH:MM        (25)
+         *
+         * Max:
+         *   YYYY-MM-DDTHH:mm:ss.fffffZ       (26)
+         *   YYYY-MM-DDTHH:mm:ss.fffff+HH:MM  (31)
+         */
+
+        $len = strlen($val);
+
+        // absolute minimum
+        if ($len < 20) {
+            return false;
+        }
+
+        // fixed datetime fingerprints
+        if (
+            !isset($val[19]) ||
+            $val[4]  !== '-' ||
+            $val[7]  !== '-' ||
+            $val[10] !== 'T' ||
+            $val[13] !== ':' ||
+            $val[16] !== ':'
+        ) {
+            return false;
+        }
+
+        // timezone detection
+        $hasZ = ($val[$len - 1] === 'Z');
+
+        $hasOffset = (
+            $len >= 25 &&
+            ($val[$len - 6] === '+' || $val[$len - 6] === '-') &&
+            $val[$len - 3] === ':'
+        );
+
+        if (!$hasZ && !$hasOffset) {
+            return false;
+        }
+
+        if ($hasOffset && $len > 31) {
+            return false;
+        }
+
+        if ($hasZ && $len > 26) {
+            return false;
+        }
+
+        $digitPositions = [
+            0,1,2,3,
+            5,6,
+            8,9,
+            11,12,
+            14,15,
+            17,18
+        ];
+
+        $timeEnd = $hasZ ? $len - 1 : $len - 6;
+
+        // fractional seconds
+        if ($timeEnd > 19) {
+            if ($val[19] !== '.' || $timeEnd < 21) {
+                return false;
+            }
+            for ($i = 20; $i < $timeEnd; $i++) {
+                $digitPositions[] = $i;
+            }
+        }
+
+        // timezone offset numeric digits
+        if ($hasOffset) {
+            foreach ([$len - 5, $len - 4, $len - 2, $len - 1] as $i) {
+                $digitPositions[] = $i;
+            }
+        }
+
+        foreach ($digitPositions as $i) {
+            if (!ctype_digit($val[$i])) {
+                return false;
+            }
+        }
+
+        return true;
+    }
+
+    protected function convertUTCDateToString(mixed $node): mixed
+    {
+        if ($node instanceof UTCDateTime) {
+            // Handle UTCDateTime objects
+            $node = DateTime::format($node->toDateTime());
+        } elseif (is_array($node) && isset($node['$date'])) {
+            // Handle Extended JSON format from (array) cast
+            // Format: {"$date":{"$numberLong":"1760405478290"}}
+            if (is_array($node['$date']) && isset($node['$date']['$numberLong'])) {
+                $milliseconds = (int)$node['$date']['$numberLong'];
+                $seconds = intdiv($milliseconds, 1000);
+                $microseconds = ($milliseconds % 1000) * 1000;
+                $dateTime = \DateTime::createFromFormat('U.u', $seconds . '.' . str_pad((string)$microseconds, 6, '0'));
+                if ($dateTime) {
+                    $dateTime->setTimezone(new \DateTimeZone('UTC'));
+                    $node = DateTime::format($dateTime);
+                }
+            }
+        } elseif (is_string($node)) {
+            // Already a string, validate and pass through
+            try {
+                new \DateTime($node);
+            } catch (\Exception $e) {
+                // Invalid date string, skip
+            }
+        }
+
+        return $node;
+    }
 }

src/Database/Adapter/MySQL.php

@@ -317,4 +317,9 @@ protected function getOperatorSQL(string $column, \Utopia\Database\Operator $ope
         // For all other operators, use parent implementation
         return parent::getOperatorSQL($column, $operator, $bindIndex);
     }
+
+    public function getSupportForTTLIndexes(): bool
+    {
+        return false;
+    }
 }

src/Database/Adapter/Pool.php

@@ -210,7 +210,7 @@ public function renameIndex(string $collection, string $old, string $new): bool
         return $this->delegate(__FUNCTION__, \func_get_args());
     }
 
-    public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = []): bool
+    public function createIndex(string $collection, string $id, string $type, array $attributes, array $lengths, array $orders, array $indexAttributeTypes = [], array $collation = [], int $ttl = 1): bool
     {
         return $this->delegate(__FUNCTION__, \func_get_args());
     }
@@ -649,4 +649,9 @@ public function getSupportForAlterLocks(): bool
     {
         return $this->delegate(__FUNCTION__, \func_get_args());
     }
+
+    public function getSupportForTTLIndexes(): bool
+    {
+        return $this->delegate(__FUNCTION__, \func_get_args());
+    }
 }