Now PyMOL within operator works. The key was to replace xs['max-radius'] -> xs['max-radius'](ctx) in table.ts

src/mol-script/runtime/query/table.ts

@@ -217,7 +217,7 @@ const symbols = [
         query: xs[0] as any,
         target: xs['target'] as any,
         minRadius: xs['min-radius'] as any,
-        maxRadius: xs['max-radius'] as any,
+        maxRadius: xs['max-radius'](ctx) as any,
         elementRadius: xs['atom-radius'] as any,
         invert: xs['invert'] as any
     })(ctx)),

src/mol-script/transpilers/pymol/operators.ts

@@ -95,7 +95,7 @@ export const operators: OperatorList = [
             .map((x: any) => parseFloat(x)),
         map: (distance: number, target: Expression) => {
             return B.struct.filter.within({
-                '0': B.struct.generator.atomGroups(),
+                '0': B.struct.generator.all(),
                 target,
                 'atom-radius': B.acp('vdw'),
                 'max-radius': distance,
@@ -116,7 +116,7 @@ export const operators: OperatorList = [
         map: (radius: number, target: Expression) => {
             return B.struct.modifier.exceptBy({
                 '0': B.struct.filter.within({
-                    '0': B.struct.generator.atomGroups(),
+                    '0': B.struct.generator.all(),
                     target,
                     'max-radius': radius,
                 }),