Changes to TRT-LLM download tool for multigpu distributed case

[apbose]

TRT-LLM installation tool for distributed

1. The download is to be done by only one GPU to avoid unnecessary downloads
2. Use of lock files in the tool for the above purpose

[github-actions]

There are some changes that do not conform to Python style guidelines:

--- /home/runner/work/TensorRT/TensorRT/py/torch_tensorrt/dynamo/utils.py	2025-09-22 06:35:28.523784+00:00
+++ /home/runner/work/TensorRT/TensorRT/py/torch_tensorrt/dynamo/utils.py	2025-09-22 06:36:00.657186+00:00
@@ -863,6 +863,6 @@
    return False


def is_thor() -> bool:
    if torch.cuda.get_device_capability() in [(11, 0)]:
-        return True
\ No newline at end of file
+        return True

[github-actions]

There are some changes that do not conform to Python style guidelines:

--- /home/runner/work/TensorRT/TensorRT/py/torch_tensorrt/dynamo/distributed/utils.py	2025-09-25 19:33:28.176615+00:00
+++ /home/runner/work/TensorRT/TensorRT/py/torch_tensorrt/dynamo/distributed/utils.py	2025-09-25 19:34:02.325958+00:00
@@ -100,11 +100,10 @@
        return True

    except Exception as e:
        logger.warning(f"Failed to detect CUDA version: {e}")
        return False
-

    return True


def _cache_root() -> Path:

[github-actions]

There are some changes that do not conform to Python style guidelines:

--- /home/runner/work/TensorRT/TensorRT/tests/py/dynamo/distributed/test_nccl_ops.py	2025-10-13 18:42:31.890493+00:00
+++ /home/runner/work/TensorRT/TensorRT/tests/py/dynamo/distributed/test_nccl_ops.py	2025-10-13 18:43:14.026641+00:00
@@ -23,10 +23,11 @@
if not dist.is_initialized():
    dist.init_process_group(
        backend="nccl",
        init_method="env://",
    )
+

class DistributedGatherModel(nn.Module):
    def __init__(self, input_dim, world_size, group_name):
        super().__init__()
        self.fc = nn.Linear(input_dim, input_dim)

[narendasan]

Why would it matter what the examples need? This is supposed to be user facing code.

[narendasan]

Where is this logger used? Typically this is a user's responsibility. we should not be creating the actual logger handlers in our library. We can do this in the examples code tho

[apbose]

ok was meant for the use case that if we want separate log outputs for the two different GPUs. But I agree these all can be moved to the user facing code

[github-actions]

There are some changes that do not conform to Python style guidelines:

--- /home/runner/work/TensorRT/TensorRT/.github/scripts/filter-matrix.py	2025-11-19 02:00:51.654228+00:00
+++ /home/runner/work/TensorRT/TensorRT/.github/scripts/filter-matrix.py	2025-11-19 02:01:22.401857+00:00
@@ -60,38 +60,40 @@
        return True


def create_distributed_config(item: Dict[str, Any]) -> Dict[str, Any]:
    """Create distributed test configuration from a regular config.
-    
+
    Takes a standard test config and modifies it for distributed testing:
    - Changes runner to multi-GPU instance
    - Adds num_gpus field
    - Adds config marker
    """
    import sys
-    
+
    # Create a copy to avoid modifying the original
    dist_item = item.copy()
-    
+
    # Debug: Show original config
    print(f"[DEBUG] Creating distributed config from:", file=sys.stderr)
    print(f"[DEBUG]   Python: {item.get('python_version')}", file=sys.stderr)
    print(f"[DEBUG]   CUDA: {item.get('desired_cuda')}", file=sys.stderr)
-    print(f"[DEBUG]   Original runner: {item.get('validation_runner')}", file=sys.stderr)
-    
+    print(
+        f"[DEBUG]   Original runner: {item.get('validation_runner')}", file=sys.stderr
+    )
+
    # Override runner to use multi-GPU instance
    dist_item["validation_runner"] = "linux.g4dn.12xlarge.nvidia.gpu"
-    
+
    # Add distributed-specific fields
    dist_item["num_gpus"] = 2
    dist_item["config"] = "distributed"
-    
+
    # Debug: Show modified config
    print(f"[DEBUG]   New runner: {dist_item['validation_runner']}", file=sys.stderr)
    print(f"[DEBUG]   GPUs: {dist_item['num_gpus']}", file=sys.stderr)
-    
+
    return dist_item


def main(args: list[str]) -> None:
    parser = argparse.ArgumentParser()
@@ -131,38 +133,43 @@
        raise ValueError(f"Invalid matrix structure: {e}")

    includes = matrix_dict["include"]
    filtered_includes = []
    distributed_includes = []  # NEW: separate list for distributed configs
-    
+
    print(f"[DEBUG] Processing {len(includes)} input configs", file=sys.stderr)

    for item in includes:
        if filter_matrix_item(
            item,
            options.jetpack == "true",
            options.limit_pr_builds == "true",
        ):
            filtered_includes.append(item)
-            
+
            # NEW: Create distributed variant for specific configs
            # Only Python 3.10 + CUDA 13.0 for now
            if item["python_version"] == "3.10" and item["desired_cuda"] == "cu130":
-                print(f"[DEBUG] Creating distributed config for py3.10+cu130", file=sys.stderr)
+                print(
+                    f"[DEBUG] Creating distributed config for py3.10+cu130",
+                    file=sys.stderr,
+                )
                distributed_includes.append(create_distributed_config(item))
-    
+
    # Debug: Show summary
    print(f"[DEBUG] Final counts:", file=sys.stderr)
    print(f"[DEBUG]   Regular configs: {len(filtered_includes)}", file=sys.stderr)
-    print(f"[DEBUG]   Distributed configs: {len(distributed_includes)}", file=sys.stderr)
+    print(
+        f"[DEBUG]   Distributed configs: {len(distributed_includes)}", file=sys.stderr
+    )

    # NEW: Output both regular and distributed configs
    filtered_matrix_dict = {
        "include": filtered_includes,
-        "distributed_include": distributed_includes  # NEW field
+        "distributed_include": distributed_includes,  # NEW field
    }
-    
+
    # Output to stdout (consumed by GitHub Actions)
    print(json.dumps(filtered_matrix_dict))


if __name__ == "__main__":