diff --git a/cmd/doc/main.go b/cmd/doc/main.go index 44b70e3..97311f7 100644 --- a/cmd/doc/main.go +++ b/cmd/doc/main.go @@ -37,7 +37,7 @@ import ( flag "github.com/spf13/pflag" "github.com/unrolled/render" "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions" - v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "sigs.k8s.io/yaml" ) @@ -109,13 +109,14 @@ type docData struct { } type orgData struct { - Page pageData - Repo string - Tag string - At string - Tags []string - CRDs map[string]models.RepoCRD - Total int + Page pageData + Server string + Repo string + Tag string + At string + Tags []string + CRDs map[string]models.RepoCRD + Total int } type homeData struct { @@ -193,10 +194,10 @@ func start() { staticHandler := http.StripPrefix("/static/", http.FileServer(http.Dir("./static/"))) r.HandleFunc("/", home) r.PathPrefix("/static/").Handler(staticHandler) - r.HandleFunc("/github.com/{org}/{repo}@{tag}", org) - r.HandleFunc("/github.com/{org}/{repo}", org) - r.HandleFunc("/raw/github.com/{org}/{repo}@{tag}", raw) - r.HandleFunc("/raw/github.com/{org}/{repo}", raw) + r.HandleFunc("/{server}/{org}/{repo}@{tag}", org) + r.HandleFunc("/{server}/{org}/{repo}", org) + r.HandleFunc("/raw/{server}/{org}/{repo}@{tag}", raw) + r.HandleFunc("/raw/{server}/{org}/{repo}", raw) r.PathPrefix("/").HandlerFunc(doc) log.Fatal(http.ListenAndServe(":5000", r)) } @@ -213,11 +214,12 @@ func home(w http.ResponseWriter, r *http.Request) { func raw(w http.ResponseWriter, r *http.Request) { parameters := mux.Vars(r) + server := parameters["server"] org := parameters["org"] repo := parameters["repo"] tag := parameters["tag"] - fullRepo := fmt.Sprintf("%s/%s/%s", "github.com", org, repo) + fullRepo := fmt.Sprintf("%s/%s/%s", server, org, repo) var rows pgx.Rows var err error if tag == "" { @@ -285,11 +287,12 @@ func raw(w http.ResponseWriter, r *http.Request) { func org(w http.ResponseWriter, r *http.Request) { parameters := mux.Vars(r) + server := parameters["server"] org := parameters["org"] repo := parameters["repo"] tag := parameters["tag"] pageData := getPageData(r, fmt.Sprintf("%s/%s", org, repo), false) - fullRepo := fmt.Sprintf("%s/%s/%s", "github.com", org, repo) + fullRepo := fmt.Sprintf("%s/%s/%s", server, org, repo) b := &pgx.Batch{} if tag == "" { b.Queue("SELECT t.name, c.group, c.version, c.kind FROM tags t INNER JOIN crds c ON (c.tag_id = t.id) WHERE LOWER(t.repo)=LOWER($1) AND t.id = (SELECT id FROM tags WHERE LOWER(repo) = LOWER($1) ORDER BY time DESC LIMIT 1);", fullRepo) @@ -348,9 +351,10 @@ func org(w http.ResponseWriter, r *http.Request) { } if len(tags) == 0 || (!tagExists && tag != "") { tryIndex(models.GitterRepo{ - Org: org, - Repo: repo, - Tag: tag, + Server: server, + Org: org, + Repo: repo, + Tag: tag, }, gitterChan) if err := page.HTML(w, http.StatusOK, "new", baseData{Page: pageData}); err != nil { log.Printf("newTemplate.Execute(): %v", err) @@ -362,12 +366,13 @@ func org(w http.ResponseWriter, r *http.Request) { foundTag = tags[0] } if err := page.HTML(w, http.StatusOK, "org", orgData{ - Page: pageData, - Repo: strings.Join([]string{org, repo}, "/"), - Tag: foundTag, - Tags: tags, - CRDs: repoCRDs, - Total: len(repoCRDs), + Page: pageData, + Server: server, + Repo: strings.Join([]string{org, repo}, "/"), + Tag: foundTag, + Tags: tags, + CRDs: repoCRDs, + Total: len(repoCRDs), }); err != nil { log.Printf("orgTemplate.Execute(): %v", err) fmt.Fprint(w, "Unable to render org template.") @@ -380,14 +385,14 @@ func doc(w http.ResponseWriter, r *http.Request) { var schema *apiextensions.CustomResourceValidation crd := &apiextensions.CustomResourceDefinition{} log.Printf("Request Received: %s\n", r.URL.Path) - org, repo, group, kind, version, tag, err := parseGHURL(r.URL.Path) + server, org, repo, group, kind, version, tag, err := parseGHURL(r.URL.Path) if err != nil { log.Printf("failed to parse Github path: %v", err) fmt.Fprint(w, "Invalid URL.") return } pageData := getPageData(r, fmt.Sprintf("%s.%s/%s", kind, group, version), false) - fullRepo := fmt.Sprintf("%s/%s/%s", "github.com", org, repo) + fullRepo := fmt.Sprintf("%s/%s/%s", server, org, repo) var c pgx.Row if tag == "" { c = db.QueryRow(context.Background(), "SELECT t.name, c.data::jsonb FROM tags t INNER JOIN crds c ON (c.tag_id = t.id) WHERE LOWER(t.repo)=LOWER($1) AND t.id = (SELECT id FROM tags WHERE repo = $1 ORDER BY time DESC LIMIT 1) AND c.group=$2 AND c.version=$3 AND c.kind=$4;", fullRepo, group, version, kind) @@ -445,14 +450,14 @@ func doc(w http.ResponseWriter, r *http.Request) { } // TODO(hasheddan): add testing and more reliable parse -func parseGHURL(uPath string) (org, repo, group, version, kind, tag string, err error) { +func parseGHURL(uPath string) (server, org, repo, group, version, kind, tag string, err error) { u, err := url.Parse(uPath) if err != nil { - return "", "", "", "", "", "", err + return "", "", "", "", "", "", "", err } elements := strings.Split(strings.Trim(u.Path, "/"), "/") if len(elements) < 6 { - return "", "", "", "", "", "", errors.New("invalid path") + return "", "", "", "", "", "", "", errors.New("invalid path") } tagSplit := strings.Split(u.Path, "@") @@ -460,5 +465,5 @@ func parseGHURL(uPath string) (org, repo, group, version, kind, tag string, err tag = tagSplit[1] } - return elements[1], elements[2], elements[3], elements[4], strings.Split(elements[5], "@")[0], tag, nil + return elements[0], elements[1], elements[2], elements[3], elements[4], strings.Split(elements[5], "@")[0], tag, nil } diff --git a/cmd/gitter/main.go b/cmd/gitter/main.go index 24b38c6..95d1d29 100644 --- a/cmd/gitter/main.go +++ b/cmd/gitter/main.go @@ -20,6 +20,16 @@ import ( "bytes" "context" "fmt" + "github.com/crdsdev/doc/pkg/crd" + "github.com/crdsdev/doc/pkg/models" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + gogithttp "github.com/go-git/go-git/v5/plumbing/transport/http" + "github.com/jackc/pgx/v4" + "github.com/jackc/pgx/v4/pgxpool" + "github.com/pkg/errors" + "gopkg.in/square/go-jose.v2/json" + "gopkg.in/yaml.v3" "io" "io/ioutil" "log" @@ -31,16 +41,6 @@ import ( "regexp" "strings" "time" - - "github.com/crdsdev/doc/pkg/crd" - "github.com/crdsdev/doc/pkg/models" - "github.com/go-git/go-git/v5" - "github.com/go-git/go-git/v5/plumbing" - "github.com/jackc/pgx/v4" - "github.com/jackc/pgx/v4/pgxpool" - "github.com/pkg/errors" - "gopkg.in/square/go-jose.v2/json" - yaml "gopkg.in/yaml.v3" ) const ( @@ -53,8 +53,51 @@ const ( dbEnv = "PG_DB" ) +type Config struct { + Repos []RepoConfig `yaml:"repos"` +} + +func (c *Config) getAuthenticationUser(gRepo models.GitterRepo) *RepoUser { + for _, element := range c.Repos { + if element.Server == gRepo.Server && element.Name == gRepo.Org+"/"+gRepo.Repo { + return &element.User + } + } + return &RepoUser{Name: "", PwdFromEnv: ""} +} + +type RepoConfig struct { + Name string `yaml:"name"` + Server string `yaml:"server"` + User RepoUser `yaml:"user"` +} + +type RepoUser struct { + Name string `yaml:"name"` + PwdFromEnv string `yaml:"pwdFromEnv"` +} + +func readConf(filename string) (*Config, error) { + buf, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + c := &Config{} + err = yaml.Unmarshal(buf, c) + if err != nil { + return nil, fmt.Errorf("in file %q: %w", filename, err) + } + + return c, err +} + func main() { dsn := fmt.Sprintf("postgresql://%s:%s@%s:%s/%s", os.Getenv(userEnv), os.Getenv(passwordEnv), os.Getenv(hostEnv), os.Getenv(portEnv), os.Getenv(dbEnv)) + conf, err := readConf("/usr/repos.yaml") + if err != nil { + fmt.Println(err) + } conn, err := pgxpool.ParseConfig(dsn) if err != nil { panic(err) @@ -65,6 +108,7 @@ func main() { } gitter := &Gitter{ conn: pool, + conf: conf, } rpc.Register(gitter) rpc.HandleHTTP() @@ -79,6 +123,7 @@ func main() { // Gitter indexes git repos. type Gitter struct { conn *pgxpool.Pool + conf *Config } type tag struct { @@ -89,19 +134,29 @@ type tag struct { // Index indexes a git repo at the specified url. func (g *Gitter) Index(gRepo models.GitterRepo, reply *string) error { - log.Printf("Indexing repo %s/%s...\n", gRepo.Org, gRepo.Repo) - dir, err := ioutil.TempDir(os.TempDir(), "doc-gitter") if err != nil { return err } defer os.RemoveAll(dir) - fullRepo := fmt.Sprintf("%s/%s/%s", "github.com", strings.ToLower(gRepo.Org), strings.ToLower(gRepo.Repo)) + + authUser := g.conf.getAuthenticationUser(gRepo) + + var fullRepo string + + fullRepo = fmt.Sprintf("%s/%s/%s", strings.ToLower(gRepo.Server), strings.ToLower(gRepo.Org), strings.ToLower(gRepo.Repo)) + + log.Printf("Indexing repo %s...\n", fullRepo) + cloneOpts := &git.CloneOptions{ URL: fmt.Sprintf("https://%s", fullRepo), Depth: 1, Progress: os.Stdout, RecurseSubmodules: git.NoRecurseSubmodules, + Auth: &gogithttp.BasicAuth{ + Username: authUser.Name, + Password: os.Getenv(authUser.PwdFromEnv), + }, } if gRepo.Tag != "" { cloneOpts.ReferenceName = plumbing.NewTagReferenceName(gRepo.Tag) @@ -109,6 +164,7 @@ func (g *Gitter) Index(gRepo models.GitterRepo, reply *string) error { } repo, err := git.PlainClone(dir, false, cloneOpts) if err != nil { + fmt.Println(err) return err } iter, err := repo.Tags() diff --git a/deploy/gitter.yaml b/deploy/gitter.yaml index 83c63a6..479576d 100644 --- a/deploy/gitter.yaml +++ b/deploy/gitter.yaml @@ -15,6 +15,9 @@ spec: containers: - name: gitter image: crdsdev/doc-gitter:latest + volumeMounts: + - name: repo-config + mountPath: usr env: - name: REDIS_HOST valueFrom: @@ -26,4 +29,24 @@ spec: configMapKeyRef: name: doc-repos key: repos - restartPolicy: Never \ No newline at end of file + volumes: + - name: repo-config + configMap: + name: private-repos-auth + + restartPolicy: Never +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: private-repos-auth + namespace: crdsdev +data: + # Add authentication details for private Github repos and servers as shown in the example entry + repos.yaml: |- + repos: + - name: owner/private-repository + server: github.mycompany.com + user: + name: machineuser + pwdFromEnv: MACHINE_SECRET diff --git a/pkg/models/repo.go b/pkg/models/repo.go index dfc08b0..9c73175 100644 --- a/pkg/models/repo.go +++ b/pkg/models/repo.go @@ -28,7 +28,8 @@ type RepoCRD struct { // GitterRepo is the repo for gitter to index. type GitterRepo struct { - Org string - Repo string - Tag string + Server string + Org string + Repo string + Tag string } diff --git a/template/org.html b/template/org.html index c5f101d..2ed3613 100644 --- a/template/org.html +++ b/template/org.html @@ -1,19 +1,19 @@
@@ -37,14 +37,14 @@

{{ .Repo }}@{{ .Tag }}

const { html } = htmReact; const { useTable, useSortBy, useGlobalFilter } = ReactTable; - const { Repo, CRDs, Tag, At, } = JSON.parse(`{{ . }}`); + const { Server, Repo, CRDs, Tag, At, } = JSON.parse(`{{ . }}`); const data = Object.keys(CRDs).map(key => CRDs[key]); const columns = [ { Header: 'Kind', accessor: 'Kind', - Cell: ({ row: { original }, value }) => html`${value}` + Cell: ({ row: { original }, value }) => html`${value}` }, { Header: 'Group',