Skip to content

Commit ba7f956

Browse files
jasrajjasraj
committed
Add proof-of-concept for downloading and initialising kdb-common libraries from GitHub
1 parent a5bf0ae commit ba7f956

File tree

4 files changed

+177
-15
lines changed

4 files changed

+177
-15
lines changed

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
11

22
# vim undo files
33
*~
4+
5+
# require 'remote GET'
6+
.require-remote-deps/

‎src/log.q

+2-2
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@
7373
.log.pattern:()!();
7474

7575
/ When call line tracing is enabled, this list of strings can be used to remove common prefixes from the file paths. By default, if this is
76-
/ empty when the library is initialised, it will be defaulted to '.require.location.root'
76+
/ empty when the library is initialised, it will be defaulted to '.require.location.paths'
7777
.log.sourcePathExcludePrefixes:();
7878

7979

@@ -91,7 +91,7 @@
9191
];
9292

9393
if[0 = count .log.sourcePathExcludePrefixes;
94-
.log.sourcePathExcludePrefixes,:enlist 1_ string .require.location.root;
94+
.log.sourcePathExcludePrefixes,:1_/: string .require.location.paths;
9595
];
9696

9797
/ setLogger calls setLevel

‎src/require.q

+31-13
Original file line numberDiff line numberDiff line change
@@ -9,15 +9,17 @@
99
/ Table containing the state of each library loaded via require
1010
.require.loadedLibs:`lib xkey flip `lib`loaded`loadedTime`inited`initedTime`files!"SBPBP*"$\:();
1111

12-
/ Root folder to search for libraries
13-
.require.location.root:`;
12+
13+
/ Search paths for 'require' libraries. Paths will be descended through (similar to 'tree')
14+
.require.location.paths:`symbol$();
1415

1516
/ Regexs to filter discovered files
1617
/ @see .require.i.tree
1718
.require.location.ignore:("*.git";"*target");
1819

1920
/ Complete list of discovered files from the root directory
20-
.require.location.discovered:enlist`;
21+
.require.location.discovered:`symbol$();
22+
2123

2224
/ Required interface implementations for 'require' and related kdb-common libraries to function correctly
2325
.require.interfaces:`lib`ifFunc xkey flip `lib`ifFunc`implFunc!"SS*"$\:();
@@ -36,23 +38,20 @@
3638
:(::);
3739
];
3840

39-
$[null root;
40-
.require.location.root:.require.i.getCwd[];
41-
.require.location.root:root
42-
];
41+
.require.location.paths:.require.location.paths union (root; .require.i.getCwd[]) null root;
4342

4443
.require.i.setDefaultInterfaces[];
4544

4645
(.require.markLibAsLoaded;.require.markLibAsInited)@\:`require;
4746

4847
/ If file tree has already been specified, don't overwrite
49-
if[.require.location.discovered~enlist`;
50-
.require.rescanRoot[];
48+
if[0 = count .require.location.discovered;
49+
.require.rescan[];
5150
];
5251

5352
.require.i.initInterfaceLibrary[];
5453

55-
.log.if.info "Require library initialised [ Root: ",string[.require.location.root]," ]";
54+
.log.if.info ("Require library initialised [ Paths: {} ]"; .require.location.paths);
5655
};
5756

5857

@@ -83,10 +82,29 @@
8382
(.require.i.load;.require.i.init)@\:lib;
8483
};
8584

86-
.require.rescanRoot:{
87-
.require.location.discovered:.require.i.tree .require.location.root;
85+
.require.rescan:.require.rescanRoot:{
86+
.log.if.debug ("Rescanning all require library paths [ Paths: {} ]"; .require.location.paths);
87+
88+
.require.location.discovered:raze .require.i.tree each .require.location.paths;
89+
90+
.log.if.info enlist["Library files refreshed [ Paths: {} ] [ Files: {} ]"],count each .require.location`paths`discovered;
91+
};
92+
93+
.require.addPath:{[path; rescan]
94+
.log.if.info ("Adding new path to 'require' search path [ Path: {} ] [ Rescan: {} ]"; path; `no`yes rescan);
95+
96+
.require.location.paths:.require.location.paths union path;
97+
98+
if[rescan;
99+
.require.rescan[];
100+
];
101+
};
102+
103+
.require.removePath:{[path]
104+
.log.if.info ("Removing path from 'require' search path [ Path: {} ]"; path);
88105

89-
.log.if.info "Library root location refreshed [ File Count: ",string[count .require.location.discovered]," ]";
106+
.require.location.paths:.require.location.paths except path;
107+
.require.rescan[];
90108
};
91109

92110
/ Marks the specified library as loaded in the loaded libraries table. NOTE: This

‎src/rrg.q

+141
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,141 @@
1+
// Require Remote GET
2+
// Copyright (c) 2021 Jaskirat Rajasansir
3+
4+
.require.lib each `http`file;
5+
6+
// URL FORMAT -- source://user/repo:version
7+
// "latest" supported for 'version'
8+
9+
.rrg.cfg.repoUrls:(`symbol$())!();
10+
.rrg.cfg.repoUrls[`gh]:"https://api.github.com/repos/{owner}/{repo}";
11+
12+
.rrg.cfg.relativeUrls:`source`query xkey flip `source`query`relativeUrl!"SS*"$\:();
13+
.rrg.cfg.relativeUrls[``]:enlist "";
14+
.rrg.cfg.relativeUrls[`gh`releases]:enlist "releases";
15+
16+
.rrg.location.root:`;
17+
18+
19+
.rrg.init:{
20+
if[null .rrg.location.root;
21+
.rrg.location.root:` sv first[.require.location.paths],`$".require-remote-deps";
22+
];
23+
24+
.file.ensureDir .rrg.location.root;
25+
26+
existingLibs:.rrg.list[];
27+
28+
if[0 < count existingLibs;
29+
.require.addPath[;0b] each existingLibs`libRoot;
30+
.require.rescan[];
31+
];
32+
33+
.log.if.info ("Require 'Remote GET' library initialised [ Download Target: {} ] [ Current Libs: {} ]"; .rrg.location.root; count existingLibs);
34+
};
35+
36+
37+
.rrg.get:{[request]
38+
details:.rrg.i.parseGetRequest request;
39+
40+
versions:details[`url],"/",.rrg.cfg.relativeUrls[details[`source],`releases]`relativeUrl;
41+
42+
verResp:.http.get[versions; ()!()];
43+
44+
if[not `success = verResp`statusType;
45+
.log.if.error "Failed to query API for repository information [ Request: ",request," ]";
46+
'"InvalidRepositoryException";
47+
];
48+
49+
releases:select from verResp[`body] where not draft, not prerelease;
50+
51+
if[0 = count releases;
52+
.log.if.error "No published releases available for repository [ Request: ",request," ]";
53+
'"NoRepositoryVersionsException";
54+
];
55+
56+
$["latest" ~ details`version;
57+
theRelease:first releases;
58+
/ else
59+
theRelease:first select from releases where tag_name like "*",details[`version],"*"
60+
];
61+
62+
if[0 = count theRelease `tarball_url;
63+
.log.if.error "Invalid version specified (or no releases available) [ Request: ",request," ]";
64+
'"InvalidRepositoryVersionException";
65+
];
66+
67+
tarTarget:` sv .rrg.location.root,`.staging,` sv details[`repo],`tar`gz;
68+
69+
.log.if.info "Downloading repository version [ Request: ",request," ] [ Version: ",theRelease[`tag_name]," ] [ Target: ",string[tarTarget]," ]";
70+
71+
repo:.http.get[theRelease`tarball_url; ()!()];
72+
tarTarget 1: repo`body;
73+
74+
extractTarget:` sv .rrg.location.root,details`repo;
75+
76+
.log.if.info "Extracting repostiory release [ Target: ",string[extractTarget]," ]";
77+
78+
if[.type.isFolder extractTarget;
79+
.log.if.debug "Removing existing download for repository [ Request: ",request," ]";
80+
.os.run[`rmFolder; 1_ string extractTarget];
81+
];
82+
83+
.file.ensureDir extractTarget;
84+
85+
.util.system "tar -xzf ",(1_ string tarTarget)," --strip-components=1 --directory ",1_ string extractTarget;
86+
(` sv extractTarget,`RRG_VERSION) 1: theRelease`tag_name;
87+
88+
rrgTree:.file.tree extractTarget;
89+
kdbFiles:rrgTree where any rrgTree like/: "*",/:(".q"; ".k"; ".q_"; ".k_");
90+
91+
if[0 = count kdbFiles;
92+
.log.if.warn ("Downloaded repository contains no kdb+ files. Not added to path [ Repository: {} ]"; request);
93+
:(::);
94+
];
95+
96+
.log.if.info ("Repository downloaded and adding to 'require' path [ Repository: {} ] [ Files: {} ]"; request; count kdbFiles)
97+
.require.addPath extractTarget;
98+
};
99+
100+
.rrg.list:{
101+
list:flip `lib`libRoot`versionFile`version!"SSS*"$\:();
102+
list:list upsert ([] lib:.file.ls .rrg.location.root; libRoot:.file.listFolderPaths .rrg.location.root);
103+
list:update versionFile:(` sv/:libRoot,\:`RRG_VERSION) from list;
104+
list:update version:first each .ns.protectedExecute[read0;] each versionFile from list;
105+
list:delete from list where version~\:.ns.const.pExecFailure;
106+
107+
:list;
108+
};
109+
110+
.rrg.import:{[lib; repo; relPath]
111+
if[not .type.isFolder repo;
112+
'"InvalidRepositoryException";
113+
];
114+
115+
isGitRepo:not .ns.const.pExecFailure ~ first .ns.protectedExecute[`.util.system; "git rev-parse --is-inside-work-tree"];
116+
117+
if[not isGitRepo;
118+
'"InvalidRepositoryException";
119+
];
120+
121+
details:.rrg.i.parseGetRequest lib;
122+
123+
repoDetail:.http.get[details`url; ()!()];
124+
125+
if[0 = count repoDetail[`body]`clone_url;
126+
'"InvalidRepositoryException";
127+
];
128+
129+
.log.if.info "Adding remote dependency as submodule [ Lib: ",lib," ] [ Local Git Repo: ",string[repo]," ] [ Relative Path: ",string[path]," ]";
130+
131+
-1 "git submodule add ",(repoDetail[`body]`clone_url)," ",1_ string relPath;
132+
};
133+
134+
.rrg.i.parseGetRequest:{[req]
135+
parsed:.http.i.getUrlDetails req;
136+
137+
result:`source`owner`repo`version!(`$-3_ parsed`scheme; `$parsed`baseUrl; `$1_ first ":" vs parsed`path; last ":" vs parsed`path);
138+
result[`url]:.util.findAndReplace[.rrg.cfg.repoUrls result`source; ("{owner}"; "{repo}"); string result`owner`repo];
139+
140+
:result;
141+
};

0 commit comments

Comments
 (0)