Skip to content

Commit 846d3d1

Browse files
author
Mark Robinson
committed
Add config to download through the API instead of rawgit
Allows for access to private repositories
1 parent bc49f56 commit 846d3d1

File tree

3 files changed

+46
-12
lines changed

3 files changed

+46
-12
lines changed

README.md

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,19 @@ If you run cwlviewer in production, you are likely to hit the GitHub API's rate
9898

9999
OAuth tokens can be obtained using the [Github authorizations API](https://developer.github.com/v3/oauth_authorizations/).
100100

101+
## Private Repositories
102+
103+
If you wish to use cwlviewer to view private Github repositories, set
104+
105+
```
106+
githubAPI.useForDownloads = true
107+
singleFileSizeLimit = 1048575
108+
```
109+
110+
Along with an authentication method which has the privileges necessary to access the repository (see above).
111+
112+
**WARNING**: This uses the [Github Contents API](https://developer.github.com/v3/repos/contents/) to download files instead of [Rawgit](https://rawgit.com/) which will increase API calls significantly.
113+
101114
## Building and Running
102115
103116
To compile you will need [Java 8](http://www.oracle.com/technetwork/java/javase/downloads/index.html) or OpenJDK 8 (`apt install openjdk-8-jdk`),

src/main/java/org/commonwl/view/github/GitHubService.java

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919

2020
package org.commonwl.view.github;
2121

22+
import org.apache.commons.codec.binary.Base64;
2223
import org.apache.commons.io.IOUtils;
2324
import org.eclipse.egit.github.core.RepositoryCommit;
2425
import org.eclipse.egit.github.core.RepositoryContents;
@@ -60,8 +61,11 @@ public class GitHubService {
6061
private final String GITHUB_CWL_REGEX = "^https?:\\/\\/github\\.com\\/([A-Za-z0-9_.-]+)\\/([A-Za-z0-9_.-]+)\\/?(?:tree|blob)\\/([^/]+)(?:\\/(.+\\.cwl))$";
6162
private final Pattern githubCwlPattern = Pattern.compile(GITHUB_CWL_REGEX);
6263

64+
private final boolean downloadWithAPI;
65+
6366
@Autowired
64-
public GitHubService(@Value("${githubAPI.authentication}") String authSetting,
67+
public GitHubService(@Value("${githubAPI.useForDownloads}") boolean downloadWithAPI,
68+
@Value("${githubAPI.authentication}") String authSetting,
6569
@Value("${githubAPI.oauthToken}") String token,
6670
@Value("${githubAPI.username}") String username,
6771
@Value("${githubAPI.password}") String password) {
@@ -73,6 +77,7 @@ public GitHubService(@Value("${githubAPI.authentication}") String authSetting,
7377
}
7478
this.contentsService = new ContentsService(client);
7579
this.commitService = new CommitService(client);
80+
this.downloadWithAPI = downloadWithAPI;
7681
}
7782

7883
/**
@@ -120,16 +125,28 @@ public List<RepositoryContents> getContents(GithubDetails githubInfo) throws IOE
120125
* @throws IOException Any API errors which may have occurred
121126
*/
122127
public String downloadFile(GithubDetails githubInfo, String sha) throws IOException {
123-
// Download the file and return the contents
124-
// rawgit.com used to download individual files from git with the correct media type
125-
String url = String.format("https://cdn.rawgit.com/%s/%s/%s/%s", githubInfo.getOwner(),
126-
githubInfo.getRepoName(), sha, githubInfo.getPath());
127-
URL downloadURL = new URL(url);
128-
InputStream download = downloadURL.openStream();
129-
try {
130-
return IOUtils.toString(download);
131-
} finally {
132-
IOUtils.closeQuietly(download);
128+
if (downloadWithAPI) {
129+
// Download the file using the Github Contents API
130+
RepositoryId repo = new RepositoryId(githubInfo.getOwner(), githubInfo.getRepoName());
131+
List<RepositoryContents> content = contentsService.getContents(repo, githubInfo.getPath(), sha);
132+
if (content.isEmpty() || content.size() > 1) {
133+
throw new IOException("Error retrieving file content from Github API");
134+
} else {
135+
byte[] base64FileContent = Base64.decodeBase64(content.get(0).getContent());
136+
return new String(base64FileContent);
137+
}
138+
} else {
139+
// Download the file and return the contents
140+
// rawgit.com used to download individual files from git with the correct media type
141+
String url = String.format("https://cdn.rawgit.com/%s/%s/%s/%s", githubInfo.getOwner(),
142+
githubInfo.getRepoName(), sha, githubInfo.getPath());
143+
URL downloadURL = new URL(url);
144+
InputStream download = downloadURL.openStream();
145+
try {
146+
return IOUtils.toString(download);
147+
} finally {
148+
IOUtils.closeQuietly(download);
149+
}
133150
}
134151
}
135152

src/main/resources/application.properties

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ graphvizStorage = /tmp
1616
cacheDays = 1
1717

1818
# File size limit for individual files in bytes
19-
# CWL files must be lower than this, but other files in the repo may be lower and in this case will
19+
# CWL files must be lower than this, but other files in the repo may be higher and in this case will
2020
# be externally linked in the Research Object Bundle
2121
singleFileSizeLimit = 5242880
2222

@@ -39,6 +39,10 @@ githubAPI.password = password
3939
# Oauth token used if githubAPI.authentication = oauth
4040
githubAPI.oauthToken = token
4141

42+
# Use Github API for downloads rather than going through rawgit CDN
43+
# Uses up significant numbers of API calls but allows the parsing of private repos when authentication is used
44+
githubAPI.useForDownloads = false
45+
4246
#=======================
4347
# MongoDB settings
4448
#=======================

0 commit comments

Comments
 (0)