Revert "Extractor: move overlay-changes check from traverser to worker"

This reverts commit 2ed8dc7.

Author: d10c

python/extractor/semmle/traverser.py

@@ -31,6 +31,19 @@ def __init__(self, options, modulenames, logger):
                 if not os.path.exists(p) and not options.ignore_missing_modules:
                     raise FileNotFoundError("'%s' does not exist." % p)
                 self.paths.add(p)
+        # During overlay extraction, only traverse the files that were changed.
+        self.overlay_changes = None
+        if 'CODEQL_EXTRACTOR_PYTHON_OVERLAY_CHANGES' in os.environ:
+            overlay_changes_file = os.environ['CODEQL_EXTRACTOR_PYTHON_OVERLAY_CHANGES']
+            logger.info("Overlay extraction mode: only extracting files changed according to '%s'", overlay_changes_file)
+            try:
+                with open(overlay_changes_file, 'r', encoding='utf-8') as f:
+                    data = json.load(f)
+                    changed_paths = data.get('changes', [])
+                    self.overlay_changes = { os.path.abspath(p) for p in changed_paths }
+            except (IOError, ValueError) as e:
+                logger.warn("Failed to read overlay changes from '%s' (falling back to full extraction): %s", overlay_changes_file, e)
+                self.overlay_changes = None
         self.exclude_paths = set([ os.path.abspath(f) for f in options.exclude_file ])
         self.exclude = exclude_filter_from_options(options)
         self.filter = filter_from_options_and_environment(options)
@@ -49,11 +62,20 @@ def __iter__(self):
                 if mod is None:
                     self.logger.error("No module named '%s'.", name)
                     raise ExtractorFailure()
+                if self.overlay_changes is not None and mod.path not in self.overlay_changes:
+                    self.logger.debug("Skipping module '%s' as it was not changed in overlay extraction.", name)
+                    continue
                 yield mod.get_extractable()
         for path in self.paths:
+            if self.overlay_changes is not None and path not in self.overlay_changes:
+                self.logger.debug("Skipping path '%s' as it was not changed in overlay extraction.", path)
+                continue
             yield Extractable.from_path(path)
         for path in self.recurse_files:
             for modpath in self._treewalk(path):
+                if self.overlay_changes is not None and modpath not in self.overlay_changes:
+                    self.logger.debug("Skipping file '%s' as it was not changed in overlay extraction.", modpath)
+                    continue
                 yield Extractable.from_path(modpath)
         for name in self.recurse_packages:
             mod = self.finder.find(name)
@@ -67,6 +89,9 @@ def __iter__(self):
                 self.logger.error("Package '%s' does not have a path.", name)
                 raise ExtractorFailure()
             for modpath in self._treewalk(path):
+                if self.overlay_changes is not None and modpath not in self.overlay_changes:
+                    self.logger.debug("Skipping package '%s' as it was not changed in overlay extraction.", modpath)
+                    continue
                 yield Extractable.from_path(modpath)
 
     def _treewalk(self, path):

python/extractor/semmle/worker.py

@@ -11,7 +11,6 @@
 from semmle.profiling import get_profiler
 from semmle.path_rename import renamer_from_options_and_env
 from semmle.logging import WARN, recursion_error_message, internal_error_message, Logger
-from semmle.util import FileExtractable, FolderExtractable
 
 class ExtractorFailure(Exception):
     'Generic exception representing the failure of an extractor.'
@@ -20,32 +19,17 @@ class ExtractorFailure(Exception):
 
 class ModuleImportGraph(object):
 
-    def __init__(self, max_depth, logger: Logger):
+    def __init__(self, max_depth):
         self.modules = {}
         self.succ = defaultdict(set)
         self.todo = set()
         self.done = set()
         self.max_depth = max_depth
-        self.logger = logger
-
-        # During overlay extraction, only traverse the files that were changed.
-        self.overlay_changes = None
-        if 'CODEQL_EXTRACTOR_PYTHON_OVERLAY_CHANGES' in os.environ:
-            overlay_changes_file = os.environ['CODEQL_EXTRACTOR_PYTHON_OVERLAY_CHANGES']
-            logger.info("Overlay extraction mode: only extracting files changed according to '%s'", overlay_changes_file)
-            try:
-                with open(overlay_changes_file, 'r', encoding='utf-8') as f:
-                    data = json.load(f)
-                    changed_paths = data.get('changes', [])
-                    self.overlay_changes = { os.path.abspath(p) for p in changed_paths }
-            except (IOError, ValueError) as e:
-                logger.warn("Failed to read overlay changes from '%s' (falling back to full extraction): %s", overlay_changes_file, e)
-                self.overlay_changes = None
 
     def add_root(self, mod):
         self.modules[mod] = 0
         if mod not in self.done:
-            self.add_todo(mod)
+            self.todo.add(mod)
 
     def add_import(self, mod, imported):
         assert mod in self.modules
@@ -55,7 +39,7 @@ def add_import(self, mod, imported):
                 self._reduce_depth(imported, self.modules[mod] + 1)
         else:
             if self.modules[mod] < self.max_depth and imported not in self.done:
-                self.add_todo(imported)
+                self.todo.add(imported)
             self.modules[imported] = self.modules[mod] + 1
 
     def _reduce_depth(self, mod, depth):
@@ -64,7 +48,7 @@ def _reduce_depth(self, mod, depth):
         if depth > self.max_depth:
             return
         if mod not in self.done:
-            self.add_todo(mod)
+            self.todo.add(mod)
         self.modules[mod] = depth
         for imp in self.succ[mod]:
             self._reduce_depth(imp, depth+1)
@@ -77,25 +61,11 @@ def get(self):
 
     def push_back(self, mod):
         self.done.remove(mod)
-        self.add_todo(mod)
+        self.todo.add(mod)
 
     def empty(self):
         return not self.todo
 
-    def add_todo(self, mod):
-        if not self._module_in_overlay_changes(mod):
-            self.logger.debug("Skipping module '%s' as it was not changed in overlay extraction.", mod)
-            return
-        self.todo.add(mod)
-
-    def _module_in_overlay_changes(self, mod):
-        if self.overlay_changes is not None:
-            if isinstance(mod, FileExtractable):
-                return mod.path in self.overlay_changes
-            if isinstance(mod, FolderExtractable):
-                return mod.path + '/__init__.py' in self.overlay_changes
-        return True
-
 class ExtractorPool(object):
     '''Pool of worker processes running extractors'''
 
@@ -120,7 +90,7 @@ def __init__(self, outdir, archive, proc_count, options, logger: Logger):
         self.enqueued = set()
         self.done = set()
         self.requirements = {}
-        self.import_graph = ModuleImportGraph(options.max_import_depth, logger)
+        self.import_graph = ModuleImportGraph(options.max_import_depth)
         logger.debug("Source archive: %s", archive)
         self.logger = logger
         DiagnosticsWriter.create_output_dir()