Skip to content

Commit 7d31bf1

Browse files
YenTheFirstbmalehorn
YenTheFirst
authored and
bmalehorn
committed
Cache Github API calls to disk
Addresses [telia-oss#142] When this resource is configured to filter versions, it can sometimes result in a large number of repeated calls to the github API to determine whether a PR should be emitted as a version or not. In extreme cases, this can result in usage that exceeds Github's API rate limit, causing the resource to be unavailable until the rate limit resets. This change mitigates the issue by caching the results of Github API calls to disk, such that repeated calls to fetch the modified file list of Pull Requests which haven't updated since the previous call, will instead be served from the cache. As Concourse intentionally re-uses containers for Check processes over a long time period, this cache should be long-lived in most deployments. [https://concourse-ci.org/implementing-resource-types.html#resource-check] It should be noted that writing a cached response to disk can fail. The underlying library, peterbourgon/diskv, returns a result in these cases. However, the http caching library, gregjones/httpcache, silently swallows any such errors. If a permission issue continually prevents writing cached responses to disk, the Check will still proceed as normal, and this resource will behave as if caching wasn't implemented.
1 parent 9ec47e2 commit 7d31bf1

File tree

3 files changed

+35
-10
lines changed

3 files changed

+35
-10
lines changed

‎github.go

Lines changed: 18 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -9,14 +9,23 @@ import (
99
"net/url"
1010
"os"
1111
"path"
12+
"path/filepath"
1213
"strconv"
1314
"strings"
1415

15-
"github.com/google/go-github/v28/github"
16+
"github.com/google/go-github/github"
17+
"github.com/gregjones/httpcache"
18+
"github.com/gregjones/httpcache/diskcache"
1619
"github.com/shurcooL/githubv4"
1720
"golang.org/x/oauth2"
1821
)
1922

23+
// on-disk location to store cached API results in between invocations, in OS-specific temp folder.
24+
// primarily useful for the Check command, which will be re-run in the same container multiple times
25+
// the underlying disk caching library (diskv via httpcache) automatically finds-or-creates this directory.
26+
// failures to write cache are silently ignored.
27+
const diskCacheFolder = "github-api-cache"
28+
2029
// Github for testing purposes.
2130
//go:generate go run github.com/maxbrunsfeld/counterfeiter/v6 -o fakes/fake_github.go . Github
2231
type Github interface {
@@ -44,19 +53,21 @@ func NewGithubClient(s *Source) (*GithubClient, error) {
4453
return nil, err
4554
}
4655

56+
diskCachePath := filepath.Join(os.TempDir(), diskCacheFolder)
57+
cache := diskcache.New(diskCachePath)
58+
cachingTransport := httpcache.NewTransport(cache)
59+
4760
// Skip SSL verification for self-signed certificates
4861
// source: https://github.com/google/go-github/pull/598#issuecomment-333039238
49-
var ctx context.Context
5062
if s.SkipSSLVerification {
51-
insecureClient := &http.Client{Transport: &http.Transport{
63+
cachingTransport.Transport = &http.Transport{
5264
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
53-
},
5465
}
55-
ctx = context.WithValue(context.TODO(), oauth2.HTTPClient, insecureClient)
56-
} else {
57-
ctx = context.TODO()
5866
}
5967

68+
var ctx context.Context
69+
ctx = context.WithValue(context.TODO(), oauth2.HTTPClient, cachingTransport.Client())
70+
6071
client := oauth2.NewClient(ctx, oauth2.StaticTokenSource(
6172
&oauth2.Token{AccessToken: s.AccessToken},
6273
))

‎go.mod

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,14 @@ module github.com/telia-oss/github-pr-resource
22

33
require (
44
github.com/golang/protobuf v1.4.0 // indirect
5-
github.com/google/go-github/v28 v28.1.1
5+
github.com/google/btree v1.0.0 // indirect
6+
github.com/google/go-github v17.0.0+incompatible
7+
github.com/google/go-querystring v1.0.0 // indirect
8+
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79
69
github.com/maxbrunsfeld/counterfeiter/v6 v6.2.3
10+
github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
711
github.com/shurcooL/githubv4 v0.0.0-20200414012201-bbc966b061dd
12+
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e // indirect
813
github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f // indirect
914
github.com/stretchr/testify v1.3.0
1015
golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5 // indirect

‎go.sum

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,14 +12,18 @@ github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrU
1212
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
1313
github.com/golang/protobuf v1.4.0 h1:oOuy+ugB+P/kBdUnG5QaMXSIyJ1q38wWSojYCb3z5VQ=
1414
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
15+
github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
16+
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
1517
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
1618
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
1719
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
1820
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
19-
github.com/google/go-github/v28 v28.1.1 h1:kORf5ekX5qwXO2mGzXXOjMe/g6ap8ahVe0sBEulhSxo=
20-
github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM=
21+
github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
22+
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
2123
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
2224
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
25+
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 h1:+ngKgrYPPJrOjhax5N+uePQ0Fh1Z7PheYoUI/0nzkPA=
26+
github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
2327
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
2428
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
2529
github.com/joefitzgerald/rainbow-reporter v0.1.0 h1:AuMG652zjdzI0YCCnXAqATtRBpGXMcAnrajcaTrSeuo=
@@ -35,8 +39,12 @@ github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw=
3539
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
3640
github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w=
3741
github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
42+
github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo=
43+
github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
3844
github.com/onsi/gomega v1.9.0 h1:R1uwffexN6Pr340GtYRIdZmAiN4J+iw6WG4wog1DUXg=
3945
github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA=
46+
github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI=
47+
github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
4048
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
4149
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
4250
github.com/sclevine/spec v1.2.0 h1:1Jwdf9jSfDl9NVmt8ndHqbTZ7XCCPbh1jI3hkDBHVYA=
@@ -45,6 +53,7 @@ github.com/sclevine/spec v1.4.0 h1:z/Q9idDcay5m5irkZ28M7PtQM4aOISzOpj4bUPkDee8=
4553
github.com/sclevine/spec v1.4.0/go.mod h1:LvpgJaFyvQzRvc1kaDs0bulYwzC70PbiYjC4QnFHkOM=
4654
github.com/shurcooL/githubv4 v0.0.0-20200414012201-bbc966b061dd h1:EwtC+kDj8s9OKiaStPZtTv3neldOyr98AXIxvmn3Gss=
4755
github.com/shurcooL/githubv4 v0.0.0-20200414012201-bbc966b061dd/go.mod h1:hAF0iLZy4td2EX+/8Tw+4nodhlMrwN3HupfaXj3zkGo=
56+
github.com/shurcooL/go v0.0.0-20180423040247-9e1955d9fb6e/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk=
4857
github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f h1:tygelZueB1EtXkPI6mQ4o9DQ0+FKW41hTbunoXZCTqk=
4958
github.com/shurcooL/graphql v0.0.0-20181231061246-d48a9a75455f/go.mod h1:AuYgA5Kyo4c7HfUmvRGs/6rGlMMV/6B1bVnB9JxJEEg=
5059
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=

0 commit comments

Comments
 (0)