Major Cleanup to scripts, including one change to requirements for dev.

Author: taggedzi

requirements.txt

@@ -1,4 +1,6 @@
-#!/usr/bin/env python3
+# Path: scripts/build_doc_index.py
+# pylint: disable=logging-fstring-interpolation
+"""Builds a documentation index file from all .md files in a directory."""
 import argparse
 import logging
 from pathlib import Path
@@ -8,6 +10,7 @@
 project_root = script_dir.parent
 
 def setup_logging():
+    """Sets up logging with a basic configuration."""
     logging.basicConfig(
         level=logging.INFO,
         format='[%(asctime)s] %(levelname)s: %(message)s'
@@ -35,6 +38,7 @@ def build_index(docs_dir: Path, index_file: Path) -> None:
     logger.info(f'Wrote documentation index: {index_file}')
 
 def main():
+    """Main entry point."""
     parser = argparse.ArgumentParser(
         description='Build an index.md in the docs folder listing all Markdown files.'
     )

scripts/build_doc_index.py

@@ -1,54 +1,124 @@
-import os
+# Path: scripts/clean_unused_packages.py
+# pylint: disable=line-too-long,broad-exception-caught
+"""
+Search a project and attempt to remove unused packages, accounting for dependencies,
+using importlib.metadata; generate cleaned requirements and removal recommendations.
+"""
 import ast
 import subprocess
-import pkg_resources
+import sys
+import sysconfig
 from pathlib import Path
+import importlib.metadata as metadata
+
 
 def get_installed_packages():
-    return {pkg.key for pkg in pkg_resources.working_set}
+    """Get a set of all installed package distribution names (lowercased)."""
+    return {dist.metadata['Name'].lower() for dist in metadata.distributions() if 'Name' in dist.metadata}
 
-def get_imported_modules(code_root):
-    imported = set()
 
-    for file in Path(code_root).rglob("*.py"):
+def get_imported_modules(code_root: Path):
+    """Get a set of all top-level modules imported from code_root."""
+    imported_loc = set()
+    for file in code_root.rglob("*.py"):
         try:
-            with open(file, "r", encoding="utf-8") as f:
-                tree = ast.parse(f.read(), filename=str(file))
-                for node in ast.walk(tree):
-                    if isinstance(node, ast.Import):
-                        for alias in node.names:
-                            imported.add(alias.name.split('.')[0])
-                    elif isinstance(node, ast.ImportFrom):
-                        if node.module:
-                            imported.add(node.module.split('.')[0])
+            tree = ast.parse(file.read_text(encoding="utf-8"), filename=str(file))
+            for node in ast.walk(tree):
+                if isinstance(node, ast.Import):
+                    for alias in node.names:
+                        imported_loc.add(alias.name.split('.')[0].lower())
+                elif isinstance(node, ast.ImportFrom):
+                    if node.module:
+                        imported_loc.add(node.module.split('.')[0].lower())
         except Exception as e:
             print(f"⚠️ Skipping {file}: {e}")
-    return imported
+    return imported_loc
 
-def find_unneeded_packages(imported_modules, installed_packages):
-    builtin_modules = stdlib_modules()
-    safe_imports = imported_modules - builtin_modules
-    return sorted(installed_packages - safe_imports)
 
 def stdlib_modules():
-    # Includes common builtins — this list is a best-effort, not exhaustive
-    import sys
+    """Return a set of standard library module names."""
     if hasattr(sys, 'stdlib_module_names'):
-        return set(sys.stdlib_module_names)
-    else:
-        import distutils.sysconfig as sysconfig
-        stdlib = sysconfig.get_python_lib(standard_lib=True)
-        return {p.name for p in Path(stdlib).iterdir() if p.is_dir()}
+        return set(m.lower() for m in sys.stdlib_module_names)
+    stdlib_path = sysconfig.get_paths()["stdlib"]
+    return {p.name.lower() for p in Path(stdlib_path).iterdir() if p.is_dir()}
+
+
+def get_all_dependencies(packages):
+    """
+    Given an iterable of distribution names, return a set of all recursive dependencies.
+    """
+    deps = set()
+    to_process = list(packages)
+    while to_process:
+        pkg = to_process.pop()
+        try:
+            dist = metadata.distribution(pkg)
+        except metadata.PackageNotFoundError:
+            continue
+        for req in dist.requires or []:
+            # Extract distribution name
+            name = req.split(';', 1)[0].split('[')[0].split(' ')[0].split('=')[0].lower()
+            if name and name not in packages and name not in deps:
+                deps.add(name)
+                to_process.append(name)
+    return deps
+
+
+def find_unneeded_packages(imported_modules, installed_packages):
+    """Find distributions installed but not needed by imports or dependencies."""
+    builtin = stdlib_modules()
+    # Map top-level modules to distributions
+    pkg2dist = metadata.packages_distributions()
+    # Identify distributions directly used by imports
+    safe = set()
+    for mod in imported_modules:
+        if mod in builtin:
+            continue
+        for dist in pkg2dist.get(mod, []):
+            safe.add(dist.lower())
+    # Filter to installed distributions
+    safe = safe & set(installed_packages)
+    # Add recursive dependencies
+    all_needed = set(safe)
+    all_needed.update(get_all_dependencies(safe))
+    # Unused = installed minus needed
+    return sorted(installed_packages - all_needed)
+
 
 def uninstall_packages(packages):
+    """Uninstall the given list of packages via pip."""
     for pkg in packages:
-        confirm = input(f"Uninstall '{pkg}'? [y/N]: ").lower()
-        if confirm == "y":
-            subprocess.run(["pip", "uninstall", "-y", pkg])
+        confirm = input(f"Uninstall '{pkg}'? [y/N]: ").strip().lower()
+        if confirm == 'y':
+            subprocess.run([sys.executable, '-m', 'pip', 'uninstall', '-y', pkg], check=False)
 
-if __name__ == "__main__":
+
+def write_recommendations(unused, rec_file: Path):
+    """Write recommended removals to a text file."""
+    rec_file.write_text('\n'.join(unused) + '\n', encoding='utf-8')
+    print(f"📄 Recommended removals saved to: {rec_file}")
+
+
+def write_clean_requirements(unused, req_file: Path):
+    """Generate a cleaned requirements.txt and write to file."""
+    try:
+        freeze = subprocess.check_output([sys.executable, '-m', 'pip', 'freeze'], text=True)
+        lines = []
+        for line in freeze.splitlines():
+            name = line.split('==')[0].lower()
+            if name not in unused:
+                lines.append(line)
+        req_file.write_text('\n'.join(lines) + '\n', encoding='utf-8')
+        print(f"📄 Cleaned requirements saved to: {req_file}")
+    except subprocess.CalledProcessError as e:
+        print(f"❌ Failed to generate cleaned requirements: {e}")
+
+
+def main():
+    """Main function."""
     print("🔍 Scanning for unused packages...")
-    code_root = "."  # You can customize this path
+    code_root = Path('.')  # customize if needed
+
     installed = get_installed_packages()
     imported = get_imported_modules(code_root)
     unused = find_unneeded_packages(imported, installed)
@@ -57,7 +127,14 @@ def uninstall_packages(packages):
     for pkg in unused:
         print(f"  - {pkg}")
 
-    if unused:
-        uninstall = input("\n❓ Do you want to uninstall them now? [y/N]: ").lower()
-        if uninstall == "y":
-            uninstall_packages(unused)
+    # Always write recommendations and cleaned requirements
+    rec_file = Path('recommended_removals.txt')
+    req_file = Path('requirements_cleaned.txt')
+    write_recommendations(unused, rec_file)
+    write_clean_requirements(unused, req_file)
+
+    if unused and input("\n❓ Uninstall them now? [y/N]: ").strip().lower() == 'y':
+        uninstall_packages(unused)
+
+if __name__ == '__main__':
+    main()

scripts/clean_unused_packages.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python3
+# Path: scripts/generate_lessons.py
 # pylint: disable=broad-exception-caught,logging-fstring-interpolation,line-too-long
 """Generate lessons from the patterns folder."""
 import argparse

scripts/generate_lessons.py

@@ -1,4 +1,6 @@
-#!/usr/bin/env python3
+# Path: scripts/rag_chunker.py
+# pylint: disable=broad-exception-caught,logging-fstring-interpolation,line-too-long
+"""Script to generate a Markdown summary of python design patterns for us in RAG systems."""
 import sys
 import argparse
 import json
@@ -16,6 +18,7 @@
 
 
 def setup_logging():
+    """Sets up logging with a basic configuration."""
     logging.basicConfig(
         level=logging.INFO,
         format='[%(asctime)s] %(levelname)s: %(message)s'
@@ -24,16 +27,17 @@ def setup_logging():
 
 logger = setup_logging()
 
-
 # System prompt for structured JSON output
 def get_system_prompt() -> str:
+    """Returns a system prompt for the AI to use to generate a structured JSON output."""
     return (
         "You are a professional Python code summarizer. "
         "Given the Python source code, return a JSON object with two keys: "
         "`summary` (a 1-2 sentence summary) and `docstrings` (a list of strings, each representing a suggested docstring for a class or function). "
         "Respond with valid JSON only, no additional text."
     )
 
+
 def annotate_code(
     code: str,
     model: str
@@ -70,11 +74,16 @@ def annotate_code(
         logger.warning(f"Annotation failed: {e}")
         return None
 
-def write_chunk(file_path: Path, summary: str, docstrings: list, output_dir: Path) -> Path:
+
+def write_chunk(
+    file_path: Path,
+    summary: str,
+    docstrings: list,
+    output_dir: Path,
+    source_dir: Path) -> Path:
     """
     Write the annotated chunk to Markdown and return its path.
     """
-    # uses module-level source_dir set in main()
     rel = file_path.relative_to(source_dir)
     chunk_name = rel.with_suffix('').as_posix().replace('/', '_').replace('\\', '_') + '.md'
     out_path = output_dir / chunk_name
@@ -117,16 +126,17 @@ def generate_index(chunks_dir: Path, index_file: Path):
         str(index_script),
         str(chunks_dir),
         str(index_file)
-    ])
+    ], check=False)
     if result.returncode == 0:
         logger.info(f"Summary index created at {index_file}")
         return True
     else:
         logger.error("Failed to generate summary index.")
         return False
 
+
 def main():
-    global source_dir
+    """Main function."""
     parser = argparse.ArgumentParser(
         description="RAG chunker: processes Python files into Markdown chunks with retry logic."
     )
@@ -150,33 +160,35 @@ def main():
         help='Path for the summary JSON index')
     args = parser.parse_args()
 
-    # assign module-level for write_chunk
     source_dir = Path(args.source)
     output_dir = Path(args.output)
     index_file = Path(args.index)
     failures_log = output_dir / 'failed_chunks.log'
 
-    # clear or create output_dir
-    if output_dir.exists():
-        for item in output_dir.iterdir():
-            if item.is_file():
-                item.unlink()
-    else:
-        output_dir.mkdir(parents=True)
+    # Ensure the output directory exists; do not delete existing files
+    output_dir.mkdir(parents=True, exist_ok=True)
 
     py_files = sorted(source_dir.rglob('*.py'))
     logger.info(f"Found {len(py_files)} Python files to process in {source_dir}.")
 
     failures = []
     for file_path in py_files:
-        logger.info(f"Processing {file_path.relative_to(source_dir)}")
+        rel = file_path.relative_to(source_dir)
+        chunk_name = rel.with_suffix('').as_posix().replace('/', '_').replace('\\', '_') + '.md'
+        md_path = output_dir / chunk_name
+
+        if md_path.exists():
+            logger.info(f"Skipping existing chunk: {chunk_name}")
+            continue
+
+        logger.info(f"Processing {rel}")
         attempt = 1
         success = False
         while attempt <= args.retries:
             result = annotate_code(file_path.read_text(encoding='utf-8'), args.model)
             if result:
                 summary, docstrings = result
-                md_path = write_chunk(file_path, summary, docstrings, output_dir)
+                md_path = write_chunk(file_path, summary, docstrings, output_dir, source_dir)
                 missing = validate_chunk(md_path)
                 if not missing:
                     success = True
@@ -192,7 +204,7 @@ def main():
             failures.append(file_path.relative_to(source_dir))
             logger.error(f"Failed to process {file_path} after {args.retries} attempts")
 
-    # write failures log
+    # Write failures log if any
     if failures:
         with failures_log.open('w', encoding='utf-8') as lf:
             for f in failures:
@@ -201,13 +213,10 @@ def main():
     else:
         logger.info("All files processed successfully.")
 
-    # generate summary index
+    # Generate summary index
     generate_index(output_dir, index_file)
 
     sys.exit(1 if failures else 0)
 
 if __name__ == '__main__':
     main()
-
-
-

scripts/rag_chunker.py

@@ -1,10 +1,22 @@
-import os
-import re
-import json
+"""Script to generate a summary index file based on the contents of Markdown files."""
+# pylint: disable=logging-fstring-interpolation
 import argparse
+import json
+import logging
+import re
 from pathlib import Path
 from typing import List, Dict
 
+def setup_logging():
+    """Sets up logging with a basic configuration."""
+    logging.basicConfig(
+        level=logging.INFO,
+        format='[%(asctime)s] %(levelname)s: %(message)s'
+    )
+    return logging.getLogger(__name__)
+
+logger = setup_logging()
+
 def parse_front_matter(text: str) -> Dict[str, str]:
     """Extracts YAML-like front matter from markdown text."""
     front_matter = {}
@@ -48,10 +60,12 @@ def build_summary_index(chunk_dir: Path) -> List[Dict[str, str]]:
     return index
 
 def save_json(index: List[Dict[str, str]], output_path: Path):
+    """Saves the summary index as a JSON file."""
     with open(output_path, 'w', encoding='utf-8') as f:
         json.dump(index, f, indent=2)
 
 def main():
+    """Main function for building the summary index."""
     parser = argparse.ArgumentParser(description="Build a summary index from markdown chunks.")
     parser.add_argument("input", help="Directory containing markdown chunks")
     parser.add_argument("output", help="Path to save the JSON index")
@@ -62,7 +76,8 @@ def main():
 
     index = build_summary_index(chunk_dir)
     save_json(index, output_path)
-    print(f"✅ Summary index saved to: {output_path.resolve()} ({len(index)} entries)")
+    logger.info(f"Summary index saved to: {output_path.resolve()} ({len(index)} entries)")
+
 
 if __name__ == "__main__":
     main()