feat(issue-16): Add private Repo support

cmd/doc/main.go

@@ -37,7 +37,7 @@ import (
 	flag "github.com/spf13/pflag"
 	"github.com/unrolled/render"
 	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
-	v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"sigs.k8s.io/yaml"
 )
 
@@ -109,13 +109,14 @@ type docData struct {
 }
 
 type orgData struct {
-	Page  pageData
-	Repo  string
-	Tag   string
-	At    string
-	Tags  []string
-	CRDs  map[string]models.RepoCRD
-	Total int
+	Page   pageData
+	Server string
+	Repo   string
+	Tag    string
+	At     string
+	Tags   []string
+	CRDs   map[string]models.RepoCRD
+	Total  int
 }
 
 type homeData struct {
@@ -193,10 +194,10 @@ func start() {
 	staticHandler := http.StripPrefix("/static/", http.FileServer(http.Dir("./static/")))
 	r.HandleFunc("/", home)
 	r.PathPrefix("/static/").Handler(staticHandler)
-	r.HandleFunc("/github.com/{org}/{repo}@{tag}", org)
-	r.HandleFunc("/github.com/{org}/{repo}", org)
-	r.HandleFunc("/raw/github.com/{org}/{repo}@{tag}", raw)
-	r.HandleFunc("/raw/github.com/{org}/{repo}", raw)
+	r.HandleFunc("/{server}/{org}/{repo}@{tag}", org)
+	r.HandleFunc("/{server}/{org}/{repo}", org)
+	r.HandleFunc("/raw/{server}/{org}/{repo}@{tag}", raw)
+	r.HandleFunc("/raw/{server}/{org}/{repo}", raw)
 	r.PathPrefix("/").HandlerFunc(doc)
 	log.Fatal(http.ListenAndServe(":5000", r))
 }
@@ -213,11 +214,12 @@ func home(w http.ResponseWriter, r *http.Request) {
 
 func raw(w http.ResponseWriter, r *http.Request) {
 	parameters := mux.Vars(r)
+	server := parameters["server"]
 	org := parameters["org"]
 	repo := parameters["repo"]
 	tag := parameters["tag"]
 
-	fullRepo := fmt.Sprintf("%s/%s/%s", "github.com", org, repo)
+	fullRepo := fmt.Sprintf("%s/%s/%s", server, org, repo)
 	var rows pgx.Rows
 	var err error
 	if tag == "" {
@@ -285,11 +287,12 @@ func raw(w http.ResponseWriter, r *http.Request) {
 
 func org(w http.ResponseWriter, r *http.Request) {
 	parameters := mux.Vars(r)
+	server := parameters["server"]
 	org := parameters["org"]
 	repo := parameters["repo"]
 	tag := parameters["tag"]
 	pageData := getPageData(r, fmt.Sprintf("%s/%s", org, repo), false)
-	fullRepo := fmt.Sprintf("%s/%s/%s", "github.com", org, repo)
+	fullRepo := fmt.Sprintf("%s/%s/%s", server, org, repo)
 	b := &pgx.Batch{}
 	if tag == "" {
 		b.Queue("SELECT t.name, c.group, c.version, c.kind FROM tags t INNER JOIN crds c ON (c.tag_id = t.id) WHERE LOWER(t.repo)=LOWER($1) AND t.id = (SELECT id FROM tags WHERE LOWER(repo) = LOWER($1) ORDER BY time DESC LIMIT 1);", fullRepo)
@@ -348,9 +351,10 @@ func org(w http.ResponseWriter, r *http.Request) {
 	}
 	if len(tags) == 0 || (!tagExists && tag != "") {
 		tryIndex(models.GitterRepo{
-			Org:  org,
-			Repo: repo,
-			Tag:  tag,
+			Server: server,
+			Org:    org,
+			Repo:   repo,
+			Tag:    tag,
 		}, gitterChan)
 		if err := page.HTML(w, http.StatusOK, "new", baseData{Page: pageData}); err != nil {
 			log.Printf("newTemplate.Execute(): %v", err)
@@ -362,12 +366,13 @@ func org(w http.ResponseWriter, r *http.Request) {
 		foundTag = tags[0]
 	}
 	if err := page.HTML(w, http.StatusOK, "org", orgData{
-		Page:  pageData,
-		Repo:  strings.Join([]string{org, repo}, "/"),
-		Tag:   foundTag,
-		Tags:  tags,
-		CRDs:  repoCRDs,
-		Total: len(repoCRDs),
+		Page:   pageData,
+		Server: server,
+		Repo:   strings.Join([]string{org, repo}, "/"),
+		Tag:    foundTag,
+		Tags:   tags,
+		CRDs:   repoCRDs,
+		Total:  len(repoCRDs),
 	}); err != nil {
 		log.Printf("orgTemplate.Execute(): %v", err)
 		fmt.Fprint(w, "Unable to render org template.")
@@ -380,14 +385,14 @@ func doc(w http.ResponseWriter, r *http.Request) {
 	var schema *apiextensions.CustomResourceValidation
 	crd := &apiextensions.CustomResourceDefinition{}
 	log.Printf("Request Received: %s\n", r.URL.Path)
-	org, repo, group, kind, version, tag, err := parseGHURL(r.URL.Path)
+	server, org, repo, group, kind, version, tag, err := parseGHURL(r.URL.Path)
 	if err != nil {
 		log.Printf("failed to parse Github path: %v", err)
 		fmt.Fprint(w, "Invalid URL.")
 		return
 	}
 	pageData := getPageData(r, fmt.Sprintf("%s.%s/%s", kind, group, version), false)
-	fullRepo := fmt.Sprintf("%s/%s/%s", "github.com", org, repo)
+	fullRepo := fmt.Sprintf("%s/%s/%s", server, org, repo)
 	var c pgx.Row
 	if tag == "" {
 		c = db.QueryRow(context.Background(), "SELECT t.name, c.data::jsonb FROM tags t INNER JOIN crds c ON (c.tag_id = t.id) WHERE LOWER(t.repo)=LOWER($1) AND t.id = (SELECT id FROM tags WHERE repo = $1 ORDER BY time DESC LIMIT 1) AND c.group=$2 AND c.version=$3 AND c.kind=$4;", fullRepo, group, version, kind)
@@ -445,20 +450,20 @@ func doc(w http.ResponseWriter, r *http.Request) {
 }
 
 // TODO(hasheddan): add testing and more reliable parse
-func parseGHURL(uPath string) (org, repo, group, version, kind, tag string, err error) {
+func parseGHURL(uPath string) (server, org, repo, group, version, kind, tag string, err error) {
 	u, err := url.Parse(uPath)
 	if err != nil {
-		return "", "", "", "", "", "", err
+		return "", "", "", "", "", "", "", err
 	}
 	elements := strings.Split(strings.Trim(u.Path, "/"), "/")
 	if len(elements) < 6 {
-		return "", "", "", "", "", "", errors.New("invalid path")
+		return "", "", "", "", "", "", "", errors.New("invalid path")
 	}
 
 	tagSplit := strings.Split(u.Path, "@")
 	if len(tagSplit) > 1 {
 		tag = tagSplit[1]
 	}
 
-	return elements[1], elements[2], elements[3], elements[4], strings.Split(elements[5], "@")[0], tag, nil
+	return elements[0], elements[1], elements[2], elements[3], elements[4], strings.Split(elements[5], "@")[0], tag, nil
 }

cmd/gitter/main.go

@@ -20,6 +20,16 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"github.com/crdsdev/doc/pkg/crd"
+	"github.com/crdsdev/doc/pkg/models"
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing"
+	gogithttp "github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/jackc/pgx/v4"
+	"github.com/jackc/pgx/v4/pgxpool"
+	"github.com/pkg/errors"
+	"gopkg.in/square/go-jose.v2/json"
+	"gopkg.in/yaml.v3"
 	"io"
 	"io/ioutil"
 	"log"
@@ -31,16 +41,6 @@ import (
 	"regexp"
 	"strings"
 	"time"
-
-	"github.com/crdsdev/doc/pkg/crd"
-	"github.com/crdsdev/doc/pkg/models"
-	"github.com/go-git/go-git/v5"
-	"github.com/go-git/go-git/v5/plumbing"
-	"github.com/jackc/pgx/v4"
-	"github.com/jackc/pgx/v4/pgxpool"
-	"github.com/pkg/errors"
-	"gopkg.in/square/go-jose.v2/json"
-	yaml "gopkg.in/yaml.v3"
 )
 
 const (
@@ -53,8 +53,51 @@ const (
 	dbEnv       = "PG_DB"
 )
 
+type Config struct {
+	Repos []RepoConfig `yaml:"repos"`
+}
+
+func (c *Config) getAuthenticationUser(gRepo models.GitterRepo) *RepoUser {
+	for _, element := range c.Repos {
+		if element.Server == gRepo.Server && element.Name == gRepo.Org+"/"+gRepo.Repo {
+			return &element.User
+		}
+	}
+	return &RepoUser{Name: "", PwdFromEnv: ""}
+}
+
+type RepoConfig struct {
+	Name   string   `yaml:"name"`
+	Server string   `yaml:"server"`
+	User   RepoUser `yaml:"user"`
+}
+
+type RepoUser struct {
+	Name       string `yaml:"name"`
+	PwdFromEnv string `yaml:"pwdFromEnv"`
+}
+
+func readConf(filename string) (*Config, error) {
+	buf, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+
+	c := &Config{}
+	err = yaml.Unmarshal(buf, c)
+	if err != nil {
+		return nil, fmt.Errorf("in file %q: %w", filename, err)
+	}
+
+	return c, err
+}
+
 func main() {
 	dsn := fmt.Sprintf("postgresql://%s:%s@%s:%s/%s", os.Getenv(userEnv), os.Getenv(passwordEnv), os.Getenv(hostEnv), os.Getenv(portEnv), os.Getenv(dbEnv))
+	conf, err := readConf("/usr/repos.yaml")
+	if err != nil {
+		fmt.Println(err)
+	}
 	conn, err := pgxpool.ParseConfig(dsn)
 	if err != nil {
 		panic(err)
@@ -65,6 +108,7 @@ func main() {
 	}
 	gitter := &Gitter{
 		conn: pool,
+		conf: conf,
 	}
 	rpc.Register(gitter)
 	rpc.HandleHTTP()
@@ -79,6 +123,7 @@ func main() {
 // Gitter indexes git repos.
 type Gitter struct {
 	conn *pgxpool.Pool
+	conf *Config
 }
 
 type tag struct {
@@ -89,26 +134,37 @@ type tag struct {
 
 // Index indexes a git repo at the specified url.
 func (g *Gitter) Index(gRepo models.GitterRepo, reply *string) error {
-	log.Printf("Indexing repo %s/%s...\n", gRepo.Org, gRepo.Repo)
-
 	dir, err := ioutil.TempDir(os.TempDir(), "doc-gitter")
 	if err != nil {
 		return err
 	}
 	defer os.RemoveAll(dir)
-	fullRepo := fmt.Sprintf("%s/%s/%s", "github.com", strings.ToLower(gRepo.Org), strings.ToLower(gRepo.Repo))
+
+	authUser := g.conf.getAuthenticationUser(gRepo)
+
+	var fullRepo string
+
+	fullRepo = fmt.Sprintf("%s/%s/%s", strings.ToLower(gRepo.Server), strings.ToLower(gRepo.Org), strings.ToLower(gRepo.Repo))
+
+	log.Printf("Indexing repo %s...\n", fullRepo)
+
 	cloneOpts := &git.CloneOptions{
 		URL:               fmt.Sprintf("https://%s", fullRepo),
 		Depth:             1,
 		Progress:          os.Stdout,
 		RecurseSubmodules: git.NoRecurseSubmodules,
+		Auth: &gogithttp.BasicAuth{
+			Username: authUser.Name,
+			Password: os.Getenv(authUser.PwdFromEnv),
+		},
 	}
 	if gRepo.Tag != "" {
 		cloneOpts.ReferenceName = plumbing.NewTagReferenceName(gRepo.Tag)
 		cloneOpts.SingleBranch = true
 	}
 	repo, err := git.PlainClone(dir, false, cloneOpts)
 	if err != nil {
+		fmt.Println(err)
 		return err
 	}
 	iter, err := repo.Tags()

deploy/gitter.yaml

@@ -15,6 +15,9 @@ spec:
           containers:
           - name: gitter
             image: crdsdev/doc-gitter:latest
+            volumeMounts:
+              - name: repo-config
+                mountPath: usr
             env:
             - name: REDIS_HOST
               valueFrom:
@@ -26,4 +29,24 @@ spec:
                 configMapKeyRef:
                   name: doc-repos
                   key: repos
-          restartPolicy: Never
+          volumes:
+            - name: repo-config
+              configMap:
+                name: private-repos-auth
+
+          restartPolicy: Never
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: private-repos-auth
+  namespace: crdsdev
+data:
+  # Add authentication details for private Github repos and servers as shown in the example entry
+  repos.yaml: |-
+    repos:
+     - name: owner/private-repository
+       server: github.mycompany.com
+       user:
+         name: machineuser
+         pwdFromEnv: MACHINE_SECRET

pkg/models/repo.go

@@ -28,7 +28,8 @@ type RepoCRD struct {
 
 // GitterRepo is the repo for gitter to index.
 type GitterRepo struct {
-	Org  string
-	Repo string
-	Tag  string
+	Server string
+	Org    string
+	Repo   string
+	Tag    string
 }

template/org.html

@@ -1,19 +1,19 @@
 <div class="content-wrapper">
     <div class="container">
         <div class="content">
-            <h1><a href="/github.com/{{ .Repo }}@{{ .Tag }}">{{ .Repo }}@{{ .Tag }}</a></h1>
+            <h1><a href="/{{ .Server }}/{{ .Repo }}@{{ .Tag }}">{{ .Repo }}@{{ .Tag }}</a></h1>
             {{ if .Tag }}
-                <a href="https://github.com/{{ .Repo }}/tree/{{ .Tag }}"><span class="label label-primary">github.com/{{ .Repo }}/tree/{{ .Tag }}</span></a>
+                <a href="https://{{ .Server }}/{{ .Repo }}/tree/{{ .Tag }}"><span class="label label-primary">{{ .Server }}/{{ .Repo }}/tree/{{ .Tag }}</span></a>
             {{ else }}
-                <a href="https://github.com/{{ .Repo }}/tree/master"><span class="label label-primary">github.com/{{ .Repo }}/tree/master</span></a>
+                <a href="https://{{ .Server }}/{{ .Repo }}/tree/master"><span class="label label-primary">{{ .Server }}/{{ .Repo }}/tree/master</span></a>
             {{ end }}
         </div>
         <select class="form-control w-md-400 w-sm-full mb-md-10 mb-5" onchange="handleSelect(this)">
-            {{ $actual := .Tag }}{{ $repo := .Repo }}{{ range $name := .Tags }}
+            {{ $actual := .Tag }}{{ $server := .Server }}{{ $repo := .Repo }}{{ range $name := .Tags }}
                 {{ if eq $name $actual }}
-                <option value="/github.com/{{ $repo }}@{{ $name }}" selected="selected">{{ $name }}</option>
+                <option value="/{{ $server }}/{{ $repo }}@{{ $name }}" selected="selected">{{ $name }}</option>
                 {{ else }}
-                <option value="/github.com/{{ $repo }}@{{ $name }}" >{{ $name }}</option>
+                <option value="/{{ $server }}/{{ $repo }}@{{ $name }}" >{{ $name }}</option>
                 {{ end }}
             {{ end }}
           </select>
@@ -37,14 +37,14 @@ <h1><a href="/github.com/{{ .Repo }}@{{ .Tag }}">{{ .Repo }}@{{ .Tag }}</a></h1>
     const { html } = htmReact;
     const { useTable, useSortBy, useGlobalFilter  } = ReactTable;
 
-    const { Repo, CRDs, Tag, At, } = JSON.parse(`{{ . }}`);
+    const { Server, Repo, CRDs, Tag, At, } = JSON.parse(`{{ . }}`);
     const data = Object.keys(CRDs).map(key => CRDs[key]);
 
     const columns = [
         {
             Header: 'Kind',
             accessor: 'Kind',
-            Cell: ({ row: { original }, value }) => html`<a href=${`/github.com/${Repo}/${original.Group}/${original.Kind}/${original.Version}@${Tag}`}>${value}</a>`
+            Cell: ({ row: { original }, value }) => html`<a href=${`/${Server}/${Repo}/${original.Group}/${original.Kind}/${original.Version}@${Tag}`}>${value}</a>`
         },
         {
             Header: 'Group',