Improve extract performance via ignoring directories early during os.walk

Co-authored-by: Steven Kao <st.kao.05@gmail.com>

Author: akx

babel/messages/extract.py

@@ -23,6 +23,7 @@
 import os
 import sys
 import tokenize
+import warnings
 from collections.abc import (
     Callable,
     Collection,
@@ -114,7 +115,33 @@ def _strip(line: str):
     comments[:] = [_strip(c) for c in comments]
 
 
-def default_directory_filter(dirpath: str | os.PathLike[str]) -> bool:
+def make_default_directory_filter(
+    method_map: Iterable[tuple[str, str]],
+    root_dir: str | os.PathLike[str],
+):
+    def directory_filter(dirpath: str | os.PathLike[str]) -> bool:
+        subdir = os.path.basename(dirpath)
+        # Legacy default behavior: ignore dot and underscore directories
+        if subdir.startswith('.') or subdir.startswith('_'):
+            return False
+
+        dir_rel = os.path.relpath(dirpath, root_dir).replace(os.sep, '/')
+
+        for pattern, method in method_map:
+            if method == "ignore" and pathmatch(pattern, dir_rel):
+                return False
+
+        return True
+
+    return directory_filter
+
+
+def default_directory_filter(dirpath: str | os.PathLike[str]) -> bool:  # pragma: no cover
+    warnings.warn(
+        "`default_directory_filter` is deprecated and will be removed in a future version of Babel.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
     subdir = os.path.basename(dirpath)
     # Legacy default behavior: ignore dot and underscore directories
     return not (subdir.startswith('.') or subdir.startswith('_'))
@@ -201,13 +228,19 @@ def extract_from_dir(
     """
     if dirname is None:
         dirname = os.getcwd()
+
     if options_map is None:
         options_map = {}
+
+    dirname = os.path.abspath(dirname)
+
     if directory_filter is None:
-        directory_filter = default_directory_filter
+        directory_filter = make_default_directory_filter(
+            method_map=method_map,
+            root_dir=dirname,
+        )
 
-    absname = os.path.abspath(dirname)
-    for root, dirnames, filenames in os.walk(absname):
+    for root, dirnames, filenames in os.walk(dirname):
         dirnames[:] = [
             subdir for subdir in dirnames if directory_filter(os.path.join(root, subdir))
         ]
@@ -224,7 +257,7 @@ def extract_from_dir(
                 keywords,
                 comment_tags,
                 strip_comment_tags,
-                dirpath=absname,
+                dirpath=dirname,
             )
 
 

tests/messages/frontend/test_extract.py

@@ -213,9 +213,16 @@ def test_extraction_with_mapping_file(self):
 
     @freeze_time("1994-11-11")
     def test_extraction_with_mapping_dict(self):
+        self._test_extraction_with_mapping_dict(ignore_pattern='**/ignored/**.*')
+
+    @freeze_time("1994-11-11")
+    def test_extraction_with_entire_directory_ignored(self):
+        self._test_extraction_with_mapping_dict(ignore_pattern='ignored')
+
+    def _test_extraction_with_mapping_dict(self, *, ignore_pattern):
         self.dist.message_extractors = {
             'project': [
-                ('**/ignored/**.*', 'ignore', None),
+                (ignore_pattern, 'ignore', None),
                 ('**.py', 'python', None),
             ],
         }