[dynamo] Add test for returning a nested recursive function and update documentation (pytorch#141713)

Addresses pytorch#137905 (comment).

Pull Request resolved: pytorch#141713
Approved by: https://github.com/jansel

Author: StrongerXi

test/dynamo/test_functions.py

@@ -3646,6 +3646,18 @@ def fn(a, b):
         self.assertIsInstance(it2, enumerate)
         self.assertEqual(list(it1), list(it2))
 
+    def test_returning_recursive_func(self):
+        @torch.compile(backend="eager", fullgraph=True)
+        def run(x):
+            def f():
+                return f
+
+            return x + 1, f
+
+        res, f = run(torch.zeros(1))
+        self.assertTrue(same(res, torch.ones(1)))
+        self.assertTrue(f is f())
+
 
 def udf_mul(x, y):
     return x * y

torch/_dynamo/side_effects.py

@@ -348,12 +348,10 @@ def track_tensor_variables_from_runahead_side_effects(self, other):
                 self.track_object_existing(other_item, other_variable)
 
     def prune_dead_object_new(self, tx):
+        # Avoid VT cycles from e.g., recursive function.
+        visited: Set[VariableTracker] = set()
         live_new_objects: Set[VariableTracker] = set()
 
-        # use this to avoid cycles in mutation_type (though I'm not sure if that
-        # can actually happen).
-        visited: Set[VariableTracker] = set({})
-
         def visit(var: VariableTracker):
             if var in visited:
                 return