Skip to content

LDEV-171 add lucee docs basic export for AI / LLM RAG #1491

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

LDEV-171 add lucee docs basic export for AI / LLM RAG #1491

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions api/build/BuildRunner.cfc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@ component accessors=true {
var builders = [];

for( var dir in dirs ){
if ( dir.type == "dir"
&& FileExists( dir.directory & "/#dir.name#/Builder.cfc" )
if ( dir.type == "dir"
&& FileExists( dir.directory & "/#dir.name#/Builder.cfc" )
&& dir.name neq "dash" ) {
builders.append( dir.name );
}
Expand Down Expand Up @@ -88,14 +88,18 @@ component accessors=true {

arguments.builder.injectMethod = this.injectMethod;

arguments.builder.injectMethod( "renderLinks", function( required string text ){
return new api.rendering.WikiLinksRenderer( docTree=variables.docTree ).renderLinks( text=arguments.text, builder=variables._builder );
arguments.builder.injectMethod( "renderLinks", function( required string text, required struct args ){
return new api.rendering.WikiLinksRenderer( docTree=variables.docTree ).renderLinks(
text = arguments.text,
builder = variables._builder,
args = arguments.args
);
} );
arguments.builder.injectMethod( "renderTemplate", function( required string template, struct args={} ){
var renderer = new api.rendering.TemplateRenderer();
var rendered = renderer.render( argumentCollection=arguments, template=_rootPathForRenderer & arguments.template );

return builder.renderLinks( rendered );
return builder.renderLinks( rendered, args );
} );

StructDelete( arguments.builder, "injectMethod" );
Expand Down
6 changes: 4 additions & 2 deletions api/rendering/WikiLinksRenderer.cfc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,16 @@ component accessors=true {

property name="docTree";

public string function renderLinks( required string text, required any builder ) {
public string function renderLinks( required string text, required any builder, required struct args ) {
var rendered = arguments.text;
var link = "";
var startPos = 1;
do {
link = _getNextLink( rendered, startPos );
if ( !IsNull( link ) ) {
rendered = Replace( rendered, link.rawMatch, arguments.builder.renderLink( link.page ?: NullValue(), link.title ), "all" );
rendered = Replace( rendered, link.rawMatch,
arguments.builder.renderLink( link.page ?: NullValue(), link.title, args ),
"all" );
startPos = link.nextStartPos;
}
} while( !IsNull( link ) );
Expand Down
232 changes: 186 additions & 46 deletions builders/html/Builder.cfc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,41 +1,42 @@
component {
public string function renderLink( any page, required string title ) {
if ( IsNull( arguments.page ) ) {
if (arguments.title.left(4) eq "http"){
return '<a href="#arguments.title#">#HtmlEditFormat( arguments.title )#</a>';
} else {
request.logger (text="Missing docs link: [[#HtmlEditFormat( arguments.title )#]]", type="WARN");
return '<a class="missing-link" title="missing link">#HtmlEditFormat( arguments.title )#</a>';
}
}
var link = arguments.page.getPath() & ".html";
return '<a href="#link#">#HtmlEditFormat( arguments.title )#</a>';
}

public string function _getIssueTrackerLink(required string name) {
var link = Replace( new api.build.BuildProperties().getIssueTrackerLink(), "{search}", urlEncodedFormat(arguments.name) )
return '<a href="#link#" class="no-oembed" target="_blank">Search Issue Tracker <i class="fa fa-external-link"></i></a>';
}

public string function _getTestCasesLink(required string name) {
var link = Replace( new api.build.BuildProperties().getTestCasesLink(), "{search}", urlEncodedFormat(arguments.name) )
return '<a href="#link#" class="no-oembed" target="_blank">Search Lucee Test Cases <i class="fa fa-external-link"></i></a> (good for further, detailed examples)';
}

public void function build( required any docTree, required string buildDirectory, required numeric threads) {
var pagePaths = arguments.docTree.getPageCache().getPages();

request.filesWritten = 0;
request.filesToWrite = StructCount(pagePaths);

var simpleBuildDirectory = arguments.buildDirectory & "Basic";

// purge previous build directory contents
loop array="#[buildDirectory, simpleBuildDirectory]#" item="local.dir" {
if ( directoryExists( dir ) )
DirectoryDelete(dir, true);
directoryCreate( dir );
}
request.logger (text="Builder HTML directory: #arguments.buildDirectory#");
request.logger (text="Builder Simple HTML directory: #simplebuildDirectory#");

new api.parsers.ParserFactory().getMarkdownParser(); // so the parser in use shows up in logs

//for ( var path in pagePaths ) {
each(pagePaths, function(path){
var tick = getTickCount();
_writePage( pagePaths[arguments.path].page, buildDirectory, docTree );
var page = pagePaths[ arguments.path ].page;
// write out full html page
var pageContent = renderPageContent( page, docTree, false, {} );
_writePage( page, buildDirectory, docTree, pageContent, {} );

// write out reduced stripped down basic html page with no navigation etc
var basicArgs = {
//"no_css": true,
"no_google_analytics": true,
"no_navigation": true,
"mainTemplate": "main_basic.cfm",
"base_href": _calcRelativeBaseHref( page, simpleBuildDirectory )
};
var basicPageContent = renderPageContent( page, docTree, false, basicArgs );
_writePage( page, simpleBuildDirectory, docTree, basicPageContent, basicArgs );

request.filesWritten++;
if ((request.filesWritten mod 100) eq 0){
Expand All @@ -51,13 +52,19 @@ component {
_copyStaticAssets( arguments.buildDirectory );
_copySiteImages( arguments.buildDirectory, arguments.docTree );
_writeSearchIndex( arguments.docTree, arguments.buildDirectory );

_copyStaticAssets( simpleBuildDirectory);
_zipBasicPages( arguments.buildDirectory, simpleBuildDirectory, "lucee-docs-basic" );
}

public string function renderPage( required any page, required any docTree, required boolean edit ){
public string function renderPageContent( required any page, required any docTree,
required boolean edit, required struct htmlOpts ){
try {
var renderedPage = renderTemplate(
template = "templates/#_getPageLayoutFile( arguments.page )#.cfm"
, args = { page = arguments.page, docTree=arguments.docTree, edit=arguments.edit }
var contentArgs = { page = arguments.page, docTree=arguments.docTree, edit=arguments.edit, htmlOpts=arguments.htmlOpts };

var pageContent = renderTemplate(
template = "templates/#_getPageLayoutFile( arguments.page )#.cfm"
, args = contentArgs
, helpers = "/builders/html/helpers"
);
} catch( any e ) {
Expand All @@ -66,7 +73,13 @@ component {
e.additional.luceeDocsPageId = arguments.page.getid();
rethrow;
}
var crumbs = arguments.docTree.getPageBreadCrumbs(arguments.page);
return pageContent;
}

public string function renderPage( required any page, required any docTree,
required string pageContent, required boolean edit, required struct htmlOptions ){

var crumbs = arguments.docTree.getPageBreadCrumbs( arguments.page );
var excludeLinkMap = {}; // tracks links to exclude from See also
var links = [];
var categories = [];
Expand Down Expand Up @@ -112,21 +125,50 @@ component {
break;
}
}

var template = arguments.htmlOptions.mainTemplate ?: "main.cfm";
var crumbsArgs = {
crumbs:crumbs,
page: arguments.page,
docTree: arguments.docTree,
categories: categories.sort("textNoCase"),
edit: arguments.edit,
htmlOpts: arguments.htmlOptions
};
var seeAlsoArgs = {
links= links,
htmlOpts=arguments.htmlOptions
}

try {

var args = {
body = Trim( arguments.pageContent )
, htmlOpts = arguments.htmlOptions
, page = arguments.page
, edit = arguments.edit
, crumbs = renderTemplate( template="layouts/breadcrumbs.cfm", helpers = "/builders/html/helpers",
args = crumbsArgs
)
, seeAlso = renderTemplate( template="layouts/seeAlso.cfm" , helpers = "/builders/html/helpers",
args = seeAlsoArgs )
};

if ( !structKeyExists(arguments.htmlOptions, "no_navigation" ) ){
args.navTree = renderTemplate( template="layouts/sideNavTree.cfm", helpers = "/builders/html/helpers", args={
crumbs=crumbs,
docTree=arguments.docTree,
pageLineage=arguments.page.getLineage(),
pageLineageMap=arguments.page.getPageLineageMap()
} );
} else {
args.navTree = "";
}

var pageContent = renderTemplate(
template = "layouts/main.cfm"
template = "layouts/#template#"
, helpers = "/builders/html/helpers"
, args = {
body = Trim( renderedPage )
, page = arguments.page
, edit = arguments.edit
, crumbs = renderTemplate( template="layouts/breadcrumbs.cfm", helpers = "/builders/html/helpers", args={ crumbs=crumbs, page=arguments.page, docTree=arguments.docTree, categories=categories.sort("textNoCase"), edit= arguments.edit } )
, navTree = renderTemplate( template="layouts/sideNavTree.cfm", helpers = "/builders/html/helpers", args={
crumbs=crumbs, docTree=arguments.docTree, pageLineage=arguments.page.getLineage(), pageLineageMap=arguments.page.getPageLineageMap()
} )
, seeAlso = renderTemplate( template="layouts/seeAlso.cfm" , helpers = "/builders/html/helpers",
args={ links=links } )
}
, args = args
);
} catch( any e ) {
//e.additional.luceeDocsPage = arguments.page;
Expand Down Expand Up @@ -173,18 +215,24 @@ component {
}

// PRIVATE HELPERS
private void function _writePage( required any page, required string buildDirectory, required any docTree ) {
private void function _writePage( required any page, required string buildDirectory,
required any docTree, required string pageContent, required struct htmlOptions ) {
var filePath = variables._getHtmlFilePath( arguments.page, arguments.buildDirectory );
var fileDirectory = GetDirectoryFromPath( filePath );

//var starttime = getTickCount();
lock name="CreateDirectory" timeout=10 {
if ( !DirectoryExists( fileDirectory ) ) {
DirectoryCreate( fileDirectory );
}
}
var pageContent = variables.cleanHtml(variables.renderPage( arguments.page, arguments.docTree, false ));
FileWrite( filePath, pageContent );

var html = variables.cleanHtml(
variables.renderPage( arguments.page,
arguments.docTree, arguments.pageContent, false ,
arguments.htmlOptions, arguments.buildDirectory
)
);
FileWrite( filePath, html );
}

// regex strips left over whitespace multiple new lines
Expand All @@ -201,6 +249,18 @@ component {
return arguments.buildDirectory & arguments.page.getPath() & ".html";
}

private string function _calcRelativeBaseHref( required any page, required string buildDirectory ) {
var path = arguments.page.getPath();
var depth = listLen( path, "/" );
if ( depth eq 1)
return "";
var baseHref = [];
loop times="#depth-1#" {
arrayAppend(baseHref, "..");
}
return ArrayToList(baseHref, "/");
}

private void function _copyStaticAssets( required string buildDirectory ) {
updateHighlightsCss( arguments.buildDirectory );
var subdirs = directoryList(path=GetDirectoryFromPath( GetCurrentTemplatePath() ) & "/assets", type="dir", recurse="false");
Expand All @@ -214,8 +274,10 @@ component {

private function updateHighlightsCss( required string buildDirectory ){
var highlighter = new api.rendering.Pygments();
var cssFile = path=GetDirectoryFromPath( GetCurrentTemplatePath() ) & "/assets/css/highlight.css";
fileWrite( cssFile, highlighter.getCss() );
var cssFile = GetDirectoryFromPath( GetCurrentTemplatePath() ) & "/assets/css/highlight.css";
var css = highlighter.getCss();
if ( trim( css ) neq trim( fileRead( cssFile ) ) )
fileWrite( cssFile, highlighter.getCss() ); // only update if changed
}

private void function _copySiteImages( required string buildDirectory, required any docTree ) {
Expand Down Expand Up @@ -305,4 +367,82 @@ component {
return '<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">#chr(10)#' &
ArrayToList(siteMap, chr(10) ) & '#chr(10)#</urlset>';
}

public string function renderLink( any page, required string title, required struct args ) {
if ( IsNull( arguments.page ) ) {
if (arguments.title.left(4) eq "http"){
return '<a href="#arguments.title#">#HtmlEditFormat( arguments.title )#</a>';
} else {
request.logger (text="Missing docs link: [[#HtmlEditFormat( arguments.title )#]]", type="WARN");
return '<a class="missing-link" title="missing link">#HtmlEditFormat( arguments.title )#</a>';
}
}
var link = arguments.page.getPath() & ".html";
if (!structKeyExists( args, "htmlOpts" ) ){
SystemOutput(structKeyList(args), true);
throw "zac";
}
//if ( arguments.page.getPath() contains "ormFlush" ) SystemOutput(args, true);
if ( structKeyExists( args, "htmlOpts" )
&& structKeyExists( args.htmlOpts, "base_href" ) ){
link = args.htmlOpts.base_href & link;
}

return '<a href="#link#">#HtmlEditFormat( arguments.title )#</a>';
}

public string function _getIssueTrackerLink(required string name) {
var link = Replace( new api.build.BuildProperties().getIssueTrackerLink(), "{search}", urlEncodedFormat(arguments.name) )
return '<a href="#link#" class="no-oembed" target="_blank">Search Issue Tracker <i class="fa fa-external-link"></i></a>';
}

public string function _getTestCasesLink(required string name) {
var link = Replace( new api.build.BuildProperties().getTestCasesLink(), "{search}", urlEncodedFormat(arguments.name) )
return '<a href="#link#" class="no-oembed" target="_blank">Search Lucee Test Cases <i class="fa fa-external-link"></i></a> (good for further, detailed examples)';
}

public function _zipBasicPages( buildDirectory, simpleBuildDirectory, zipName ){

var zipFilename = arguments.zipName & ".zip";
var doubleZipFilename = arguments.zipName & "-zipped.zip";

// neat trick, storing then zipping the stored zip reduces the file size from 496 Kb to 216 Kb
var tempStoredZip = getTempFile( "", "#zipfileName#-store", "zip" );
var tempDoubleZip = getTempFile( "", "#zipfileName#-normal", "zip" );
var tempNormalZip = getTempFile( "", "#zipfileName#-normal", "zip" );

zip action="zip"
source="#arguments.simpleBuildDirectory#"
file="#tempStoredZip#"
compressionmethod="store"
recurse="true";

zip action="zip"
source="#arguments.simpleBuildDirectory#"
file="#tempDoubleZip#"
compressionmethod="deflateUtra" // typo in cfzip!
recurse="false" {
zipparam entrypath="#zipFilename#" source="#tempStoredZip#";
};
fileDelete( tempStoredZip );

zip action="zip"
source="#arguments.simpleBuildDirectory#"
file="#tempNormalZip#"
recurse="true";

publishWithChecksum( tempNormalZip, "#buildDirectory#/#zipFilename#" );
publishWithChecksum( tempDoubleZip, "#buildDirectory#/#doubleZipFilename#" );
};

function publishWithChecksum( src, dest ){
request.logger (text="Builder copying zip to #dest#");
fileCopy( src, dest );
loop list="md5,sha1" item="local.hashType" {
var checksumPath = left( dest, len( dest ) - 3 ) & hashType;
filewrite( checksumPath, lcase( hash( fileReadBinary( arguments.src ), hashType ) ) );
request.logger (text="Builder added #checksumPath# checksum");
}
}

}
9 changes: 6 additions & 3 deletions builders/html/layouts/breadcrumbs.cfm
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,14 +3,17 @@
<cfparam name="args.categories" type="array" />
<cfparam name="args.docTree" type="any" />
<cfparam name="args.page" type="any" />

<cfif args.edit>
<cfset local.docs_base_url = "http://#cgi.http_host#">
<cfelse>
<cfset local.docs_base_url = "https://docs.lucee.org">
<cfif structKeyExists( args, "htmlOpts" )
&& structKeyExists( args.htmlOpts, "base_href" )>
<cfset local.docs_base_url = args.htmlOpts.base_href>
<cfelse>
<cfset local.docs_base_url = "https://docs.lucee.org">
</cfif>
</cfif>


<cfif args.page.getId() neq "/home" and ArrayLen(args.crumbs)>
<!--- pages may have multiple crumbs LD-112 --->
<cfloop array="#args.crumbs#" item="local._crumbs">
Expand Down
Loading
Loading