DEV-2308: Render Terraform Modules and GitHub Actions (#628)

* adjust titles

* Update module readme MD file with fixed URLs and content

* Update custom loaders resolved path and add method to fetch Terraform files

* Refactor terraform file fetching method for efficiency

* Fix MDX format and <details><summary> html tags

* Fix URL formatting in MDX content rendering

* Disable category.json rendering and clean up URLs

* Update Terraform docs rendering and collapse sidebar items

* Update anchor tags for inputs and outputs in Terraform docs

* Cache tmp directory and update sidebar label

* Cache rendered content for website build process

* Add logging for module provider and name, and docs directory

* Render terraform docs and README for module directory

* Update debug logging and exclude terraform-aws-components

* Add debug logging for ls commands in ModuleRenderer

* Refactor debug logging in ModuleRenderer.py

* Refactor build target in Makefile and logging in Python script

* Replace <= with &lt;= to avoid parsing issues

* Update GitHub Action edit URL and fix MDX formatting

* Update mdx formatting in rendering.py

* Update sidebar directory path for GitHub actions

* Update sidebar_label to 'actions' and add sidebar_class_name

* Remove cache files from .gitignore

* Update cache handling logic for GitHubProvider

Author: milldr

.github/actions/build-website/action.yml

@@ -52,11 +52,24 @@ runs:
       uses: actions/setup-python@v5
       with:
         python-version: '3.10'
+        cache: 'pip'
 
-    - name: "Install Python Dependencies"
+    - name: Cache rendered content
+      uses: actions/cache@v4
+      with:
+        path: |
+          .build-harness
+          .cache
+        key: ${{ github.workflow }}
+
+    - name: "Initialize Build Harness"
       shell: bash
       run: |
         make init
+
+    - name: "Install Python Dependencies"
+      shell: bash
+      run: |
         pip install -r scripts/docs-collator/requirements.txt
 
     - name: "Install terraform-docs"
@@ -91,5 +104,4 @@ runs:
         GOOGLE_TAG_MANAGER: ${{ inputs.google_tag_manager }}
         GOOGLE_SITE_VERIFICATION_ID: ${{ inputs.google_site_verification_id }}
       run: |
-        make init
         make build-production

.gitignore

@@ -7,6 +7,7 @@
 # Generated files
 .docusaurus
 .cache-loader
+.cache
 /layouts
 /public
 /resources

Makefile

@@ -10,7 +10,7 @@ all: real-clean build
 deps:
 	npm install
 
-.PHONY: build
+.PHONY: init build
 
 render:
 	./scripts/render-docs-for-components.sh

docs/github-actions/library/actions/actions.mdx

@@ -1,6 +1,7 @@
 ---
 title: GitHub Actions
-sidebar_label: Overview of GitHub Actions
+sidebar_label: actions
+sidebar_class_name: command
 description: GitHub Actions
 sidebar_position: 0
 ---

docs/github-actions/library/library.mdx

@@ -1,6 +1,6 @@
 ---
 title: GitHub Actions
-sidebar_label: Overview of GitHub Actions
+sidebar_label: actions
 sidebar_position: 0
 sidebar_class_name: hidden
 description: GitHub Actions Library

scripts/docs-collator/AbstractRenderer.py

@@ -38,6 +38,7 @@ def _post_rendering_fixes(self, repo, readme_md_file, submodule_dir=""):
         content = rendering.replace_relative_links_with_github_links(
             repo, content, submodule_dir
         )
+        content = rendering.fix_mdx_format(content)
         io.save_string_to_file(readme_md_file, content)
 
     def _copy_extra_resources_for_docs(self, module_download_dir, module_docs_dir):

scripts/docs-collator/ComponentRenderer.py

@@ -117,7 +117,7 @@ def __render_doc(self, component, file):
         io.save_string_to_file(result_file, doc_content)
 
         # Breaking builds, not sure why and not adding value. Disabling.
-        #self.__create_indexes_for_subfolder(component)
+        # self.__create_indexes_for_subfolder(component)
 
     def __create_indexes_for_subfolder(self, component):
         for root, dirs, files in os.walk(os.path.join(self.docs_dir, component)):

scripts/docs-collator/GitHubProvider.py

@@ -2,9 +2,9 @@
 import logging
 import os
 import re
-
+import pickle
 from github import Github, GithubException
-
+from functools import lru_cache
 from utils import io
 
 GITHUB_ORG = "cloudposse"
@@ -15,15 +15,27 @@
     "^github-action-.*"
 )  # convention is github-action-<NAME>
 
+CACHE_DIR = ".cache"
+CACHE_FILE = os.path.join(CACHE_DIR, "file_contents_cache.pkl")
+REPOS_CACHE_FILE = os.path.join(CACHE_DIR, "repos_cache.pkl")
+
 
 class GitHubProvider:
     def __init__(self, github_api_token):
         self.github = Github(github_api_token)
+        self.cache = self.load_cache(CACHE_FILE)
+        self.repos_cache = self.load_cache(REPOS_CACHE_FILE)
 
     def get_terraform_repos(self, includes_csv, excludes_csv):
-        return self.__get_repos(
+        key = (includes_csv, excludes_csv)
+        if key in self.repos_cache:
+            return self.repos_cache[key]
+        repos = self.__get_repos(
             includes_csv, excludes_csv, TERRAFORM_MODULE_NAME_PATTERN
         )
+        self.repos_cache[key] = repos
+        self.save_cache(REPOS_CACHE_FILE, self.repos_cache)
+        return repos
 
     def get_github_actions_repos(self, includes_csv, excludes_csv):
         return self.__get_repos(includes_csv, excludes_csv, GITHUB_ACTION_NAME_PATTERN)
@@ -75,12 +87,27 @@ def list_repo_dir(self, repo, remote_dir, recursive=True):
 
         return result
 
-    def fetch_file(self, repo, remote_file, output_dir):
-        io.create_dirs(os.path.join(output_dir, os.path.dirname(remote_file)))
+    def get_file_content(self, repo, remote_file):
+        """
+        Fetches the content of a file from a GitHub repository AND cache the result
+        """
+        key = (repo.full_name, remote_file)
+        if key in self.cache:
+            return self.cache[key]
         content_encoded = repo.get_contents(
             remote_file, ref=repo.default_branch
         ).content
         content = base64.b64decode(content_encoded)
+        self.cache[key] = content
+        self.save_cache(CACHE_FILE, self.cache)
+        return content
+
+    def fetch_file(self, repo, remote_file, output_dir):
+        io.create_dirs(os.path.join(output_dir, os.path.dirname(remote_file)))
+
+        # fetch file content, supported by caching
+        content = self.get_file_content(repo, remote_file)
+
         output_file = os.path.join(output_dir, remote_file)
         io.save_to_file(output_file, content)
         logging.info(f"Fetched file: {remote_file}")
@@ -99,3 +126,14 @@ def __is_valid(self, repo, pattern):
             return False
 
         return True
+
+    def load_cache(self, file_path):
+        if os.path.exists(file_path):
+            with open(file_path, "rb") as f:
+                return pickle.load(f)
+        return {}
+
+    def save_cache(self, file_path, cache):
+        os.makedirs(CACHE_DIR, exist_ok=True)
+        with open(file_path, "wb") as f:
+            pickle.dump(cache, f)

scripts/docs-collator/ModuleFetcher.py

@@ -32,6 +32,8 @@ def fetch(self, repo):
         if SUBMODULES_DIR in remote_files:
             self.__fetch_submodules(repo, module_download_dir)
 
+        self.__fetch_terraform_files(repo, module_download_dir, remote_files)
+
     def __fetch_images(self, repo, module_download_dir):
         remote_files = self.github_provider.list_repo_dir(repo, IMAGES_DIR)
 
@@ -59,3 +61,12 @@ def __fetch_submodules(self, repo, module_download_dir):
                     module_download_dir,
                     submodule_dir=os.path.dirname(readme_file),
                 )
+
+    def __fetch_terraform_files(self, repo, module_download_dir, remote_files):
+        for remote_file in remote_files:
+            # We only need variables and output files to render terraform-docs.
+            # _Hopefully_ the module follows the convention of having these files with all varialbes and outputs,
+            # but that is an assumption we're making here for the sake of deployment speed.
+            # if remote_file in ["variables.tf", "outputs.tf"]:
+            if remote_file.endswith(".tf"):
+                self.github_provider.fetch_file(repo, remote_file, module_download_dir)

scripts/docs-collator/ModuleRenderer.py

@@ -36,7 +36,10 @@ def render(self, repo):
         self._pre_rendering_fixes(repo, module_download_dir)
 
         provider, module_name = rendering.parse_terraform_repo_name(repo.name)
+        logging.debug(f"Provider: {provider}, Module: {module_name}")
+
         module_docs_dir = os.path.join(self.docs_dir, provider, module_name)
+        logging.debug(f"Module docs dir: {module_docs_dir}")
 
         self.__render_readme(module_download_dir, module_docs_dir)
 
@@ -52,8 +55,9 @@ def render(self, repo):
             repo, module_download_dir, module_docs_dir
         )
 
-        self.__create_index_for_provider(repo)
-        self.__create_indexes_for_subfolders(repo)
+        # Disable category.json for now
+        # self.__create_index_for_provider(repo)
+        # self.__create_indexes_for_subfolders(repo)
 
     def __render_readme(self, module_download_dir, module_docs_dir):
         readme_yaml_file = os.path.join(module_download_dir, README_YAML)
@@ -62,6 +66,21 @@ def __render_readme(self, module_download_dir, module_docs_dir):
 
         io.create_dirs(module_docs_dir)
 
+        # Re-render terraform docs with this repo's terraform-docs template for modules.
+        # This replaces docs/terraform.md for the given module in place
+        logging.debug(f"Rendering terraform docs for: {module_download_dir}")
+        rendering.render_terraform_docs(
+            module_download_dir, os.path.join(TEMPLATES_DIR, "terraform-docs.yml")
+        )
+
+        # Run the make readme command in the module directory to compile README.md
+        logging.debug(f"Rendering README.md for: {module_download_dir}")
+        logging.debug(f"make readme")
+        logging.debug(f"README_TEMPLATE_FILE: {readme_tmpl_file}")
+        logging.debug(f"README_FILE: {readme_md_file}")
+        logging.debug(f"README_YAML: {readme_yaml_file}")
+        logging.debug(f"README_TEMPLATE_YAML: {readme_yaml_file}")
+        logging.debug(f"README_INCLUDES: {module_download_dir}")
         response = subprocess.run(
             [
                 "make",
@@ -185,4 +204,5 @@ def __post_rendering_fixes_for_submodule(self, readme_md_file):
         content = io.read_file_to_string(readme_md_file)
         content = rendering.fix_self_non_closing_br_tags(content)
         content = rendering.fix_custom_non_self_closing_tags_in_pre(content)
+        content = rendering.fix_mdx_format(content)
         io.save_string_to_file(readme_md_file, content)

scripts/docs-collator/render_docs_for_modules.py

@@ -31,6 +31,11 @@ def main(
     logging.info(f"Found {len(repos)} repositories to process")
 
     for repo in repos:
+        # logging.info(f"Debugging module: {repo.full_name}")
+        # if repo.full_name != "cloudposse/terraform-aws-ec2-ami-backup":
+        #     logging.info(f"skipping...")
+        #     continue
+
         try:
             logging.info(f"Fetching files for: {repo.full_name}")
             fetcher.fetch(repo)

scripts/docs-collator/templates/components/terraform-docs.yml

@@ -4,7 +4,7 @@ content: |-
 
   {{ .Header }}
 
-  ## Contents
+  ## Terraform
 
   {{ if ne (len .Module.Requirements) 0 -}}
   ### Version Requirements
@@ -70,7 +70,7 @@ content: |-
   <dl>
   {{- range .Module.Inputs }}
   {{- if and (not (has .Name $context_variables)) .Required }}
-    <dt>`{{ .Name }}`{{ if lt (len (split "\n" (tostring .Type))) 2 }} (`{{ tostring .Type }}`){{end}} <i>required</i>{{ if contains "OBSOLETE: " (tostring .Description) }} <strong>OBSOLETE</strong>{{ end }}</dt>
+    <dt><a id="{{ .Name }}" href="#{{ .Name }}">`{{ .Name }}`</a>{{ if lt (len (split "\n" (tostring .Type))) 2 }} (`{{ tostring .Type }}`){{end}} <i>required</i>{{ if contains "OBSOLETE: " (tostring .Description) }} <strong>OBSOLETE</strong>{{ end }}</dt>
     <dd>
       {{- $lines := regexSplit "\n" (tostring .Description | replace "OBSOLETE: " "") -1 -}}
       {{- range $lines }}
@@ -100,7 +100,7 @@ content: |-
   <dl>
   {{- range .Module.Inputs }}
   {{- if and (not (has .Name $context_variables)) (not .Required) }}
-    <dt>`{{ .Name }}`{{ if lt (len (split "\n" (tostring .Type))) 2 }} (`{{ tostring .Type }}`){{end}} <i>optional</i>{{ if contains "OBSOLETE: " (tostring .Description) }} <strong>OBSOLETE</strong>{{ end }}</dt>
+    <dt><a id="{{ .Name }}" href="#{{ .Name }}">`{{ .Name }}`</a>{{ if lt (len (split "\n" (tostring .Type))) 2 }} (`{{ tostring .Type }}`){{end}} <i>optional</i>{{ if contains "OBSOLETE: " (tostring .Description) }} <strong>OBSOLETE</strong>{{ end }}</dt>
     <dd>
       {{- $lines := regexSplit "\n" (tostring .Description | replace "OBSOLETE: " "") -1 -}}
       {{- range $lines }}
@@ -114,7 +114,7 @@ content: |-
       ```
       <br/>
       {{ end }}
-      **Default value:** {{ if lt (len (split "\n" .GetValue)) 2 }}`{{ .GetValue }}`{{ else }}
+      **Default value:** {{ if lt (len (split "\n" .GetValue)) 2 }}`{{ .GetValue | replace "{}" "{ }" | replace "[]" "[ ]" }}`{{ else }}
       ```hcl
       {{- $lines := regexSplit "\n" .GetValue -1 -}}
       {{- range $lines }}
@@ -138,7 +138,7 @@ content: |-
   <dl>
   {{- range .Module.Inputs }}
   {{- if and (has .Name $context_variables) }}
-    <dt>`{{ .Name }}`{{ if lt (len (split "\n" (tostring .Type))) 2 }} (`{{ tostring .Type }}`){{end}} <i>{{ ternary .Required "required" "optional" }}</i>{{ if contains "OBSOLETE: " (tostring .Description) }} <strong>OBSOLETE</strong>{{ end }}</dt>
+    <dt><a id="{{ .Name }}" href="#{{ .Name }}">`{{ .Name }}`</a>{{ if lt (len (split "\n" (tostring .Type))) 2 }} (`{{ tostring .Type }}`){{end}} <i>{{ ternary .Required "required" "optional" }}</i>{{ if contains "OBSOLETE: " (tostring .Description) }} <strong>OBSOLETE</strong>{{ end }}</dt>
     <dd>
       {{- $lines := regexSplit "\n" (tostring .Description | replace "OBSOLETE: " "") -1 -}}
       {{- range $lines }}
@@ -152,7 +152,7 @@ content: |-
       ```
       <br/>
       {{ end }}
-      **Default value:** {{ if lt (len (split "\n" .GetValue)) 2 }}`{{ .GetValue }}`{{ else }}
+      **Default value:** {{ if lt (len (split "\n" .GetValue)) 2 }}`{{ .GetValue | replace "{}" "{ }" | replace "[]" "[ ]" }}`{{ else }}
       ```hcl
       {{- $lines := regexSplit "\n" .GetValue -1 -}}
       {{- range $lines }}
@@ -171,7 +171,7 @@ content: |-
 
   <dl>
   {{- range .Module.Outputs }}
-    <dt>`{{ .Name }}`{{ if contains "OBSOLETE: " (tostring .Description) }} <strong>OBSOLETE</strong>{{ end }}</dt>
+    <dt><a id="{{ .Name }}" href="#{{ .Name }}">`{{ .Name }}`</a>{{ if contains "OBSOLETE: " (tostring .Description) }} <strong>OBSOLETE</strong>{{ end }}</dt>
     <dd>
       {{- $lines := regexSplit "\n" (tostring .Description | replace "OBSOLETE: " "" | default "n/a" ) -1 -}}
       {{- range $lines }}

scripts/docs-collator/templates/github-actions/readme.md

@@ -15,7 +15,7 @@ description: |-
 tags:
 {{ (ds "config").tags | data.ToYAML | strings.Indent 2 -}}
 {{- end }}
-custom_edit_url: https://github.com/cloudposse/{{ $fullName }}/edit/master/README.md
+custom_edit_url: https://github.com/cloudposse/{{ $fullName }}/edit/main/README.md
 ---
 
 # GitHub Action: `{{ $shortName }}`
@@ -75,7 +75,7 @@ custom_edit_url: https://github.com/cloudposse/{{ $fullName }}/edit/master/READM
 
 {{ if has (ds "config") "include" }}
 {{ range $file := (datasource "config").include -}}
-{{ (include "includes" $file) }}
+{{ (include "includes" (printf "%s/%s" $fullName $file)) }}
 {{- end }}
 {{- end }}
 {{- end }}

scripts/docs-collator/templates/modules/readme.md

@@ -15,7 +15,7 @@ description: |-
 tags:
 {{ (ds "config").tags | data.ToYAML | strings.Indent 2 -}}
 {{- end }}
-custom_edit_url: https://github.com/cloudposse/{{ $fullModuleName }}/edit/master/README.md
+custom_edit_url: https://github.com/cloudposse/{{ $fullModuleName }}/edit/main/README.md
 ---
 
 # Module: `{{ $shortModuleName }}`
@@ -75,7 +75,7 @@ custom_edit_url: https://github.com/cloudposse/{{ $fullModuleName }}/edit/master
 
 {{ if has (ds "config") "include" }}
 {{ range $file := (datasource "config").include -}}
-{{ (include "includes" $file) }}
+{{ (include "includes" (printf "%s/%s" $fullModuleName $file)) }}
 {{- end }}
 {{- end }}
 {{- end }}