Skip to content

Commit b3c3d38

Browse files
BradLugoBradLugo
committed
jsonblob: implement Load as documented
Signed-off-by: Brad Lugo <blugo@redhat.com>
1 parent c033ce9 commit b3c3d38

5 files changed

Lines changed: 208 additions & 90 deletions

File tree

‎libvuln/jsonblob/jsonblob.go‎

Lines changed: 26 additions & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -45,85 +45,43 @@ type Store struct {
4545
}
4646

4747
// Load reads in all the records serialized in the provided [io.Reader].
48-
func Load(ctx context.Context, r io.Reader) (*Loader, error) {
49-
l := Loader{
50-
dec: json.NewDecoder(r),
51-
cur: uuid.Nil,
48+
func Load(ctx context.Context, r io.Reader) (*Store, error) {
49+
s, err := New()
50+
if err != nil {
51+
return nil, err
5252
}
53-
return &l, nil
54-
}
55-
56-
// Loader is an iterator that returns a series of [Entry].
57-
//
58-
// Users should call [*Loader.Next] until it reports false, then check for
59-
// errors via [*Loader.Err].
60-
type Loader struct {
61-
err error
62-
e *Entry
63-
64-
dec *json.Decoder
65-
next *Entry
66-
de diskEntry
67-
cur uuid.UUID
68-
}
6953

70-
// Next reports whether there's an [Entry] to be processed.
71-
func (l *Loader) Next() bool {
72-
if l.err != nil {
73-
return false
54+
l, err := NewLoader(r)
55+
if err != nil {
56+
return nil, err
7457
}
7558

76-
for l.err = l.dec.Decode(&l.de); l.err == nil; l.err = l.dec.Decode(&l.de) {
77-
id := l.de.Ref
78-
// If we just hit a new Entry, promote the current one.
79-
if id != l.cur {
80-
l.e = l.next
81-
l.next = &Entry{}
82-
l.next.Updater = l.de.Updater
83-
l.next.Fingerprint = l.de.Fingerprint
84-
l.next.Date = l.de.Date
59+
// TODO(DO NOT MERGE): ~~This implementation might be a bit naive. Currently,
60+
// it basically copies [OfflineImport]. We could probably do some custom
61+
// parsing such that it basically just decodes the json into the
62+
// appropriate [Store] fields.~~
63+
// Actually, this might be the way.
64+
// [OfflineImport]: https://github.com/quay/claircore/blob/126f688bb11220fb34708719be91952dc32ff7b1/libvuln/updates.go#L17-L74
65+
for l.Next() {
66+
if err := ctx.Err(); err != nil {
67+
return nil, err
8568
}
86-
switch l.de.Kind {
87-
case driver.VulnerabilityKind:
88-
vuln := claircore.Vulnerability{}
89-
if err := json.Unmarshal(l.de.Vuln.buf, &vuln); err != nil {
90-
l.err = err
91-
return false
69+
e := l.Entry()
70+
if e.Enrichment != nil {
71+
if _, err = s.UpdateEnrichments(ctx, e.Updater, e.Fingerprint, e.Enrichment); err != nil {
72+
return nil, fmt.Errorf("updating enrichements: %w", err)
9273
}
93-
l.next.Vuln = append(l.next.Vuln, &vuln)
94-
case driver.EnrichmentKind:
95-
en := driver.EnrichmentRecord{}
96-
if err := json.Unmarshal(l.de.Enrichment.buf, &en); err != nil {
97-
l.err = err
98-
return false
99-
}
100-
l.next.Enrichment = append(l.next.Enrichment, en)
10174
}
102-
// If this was an initial diskEntry, promote the ref.
103-
if id != l.cur {
104-
l.cur = id
105-
// If we have an Entry ready, report that.
106-
if l.e != nil {
107-
return true
75+
if e.Vuln != nil {
76+
if _, err = s.UpdateVulnerabilities(ctx, e.Updater, e.Fingerprint, e.Vuln); err != nil {
77+
return nil, fmt.Errorf("updating vulnerabilities: %w", err)
10878
}
10979
}
11080
}
111-
l.e = l.next
112-
return true
113-
}
114-
115-
// Entry returns the latest loaded [Entry].
116-
func (l *Loader) Entry() *Entry {
117-
return l.e
118-
}
119-
120-
// Err is the latest encountered error.
121-
func (l *Loader) Err() error {
122-
// Don't report EOF as an error.
123-
if errors.Is(l.err, io.EOF) {
124-
return nil
81+
if err := l.Err(); err != nil {
82+
return nil, err
12583
}
126-
return l.err
84+
return s, nil
12785
}
12886

12987
// Store writes out the contents of the receiver to the provided [io.Writer].

‎libvuln/jsonblob/jsonblob_test.go‎

Lines changed: 10 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,14 @@ package jsonblob
33
import (
44
"bytes"
55
"context"
6-
"io"
7-
"testing"
8-
6+
"fmt"
97
"github.com/google/go-cmp/cmp"
10-
"golang.org/x/sync/errgroup"
11-
128
"github.com/quay/claircore"
139
"github.com/quay/claircore/libvuln/driver"
10+
"golang.org/x/sync/errgroup"
11+
"io"
12+
"testing"
13+
1414
"github.com/quay/claircore/test"
1515
)
1616

@@ -69,24 +69,13 @@ func TestRoundtrip(t *testing.T) {
6969
eg, ctx := errgroup.WithContext(ctx)
7070
eg.Go(func() error { defer w.Close(); return a.Store(w) })
7171
eg.Go(func() error {
72-
l, err := Load(ctx, io.TeeReader(r, &buf))
72+
s, err := Load(ctx, io.TeeReader(r, &buf))
7373
if err != nil {
74-
return err
75-
}
76-
for l.Next() {
77-
e := l.Entry()
78-
if e.Vuln != nil && e.Enrichment != nil {
79-
t.Error("expecting entry to have either vulnerability or enrichment, got both")
80-
}
81-
if e.Vuln != nil {
82-
got.V = append(got.V, l.Entry().Vuln...)
83-
}
84-
if e.Enrichment != nil {
85-
got.E = append(got.E, l.Entry().Enrichment...)
86-
}
74+
return fmt.Errorf("failed to load jsonblob: %w", err)
8775
}
88-
if err := l.Err(); err != nil {
89-
return err
76+
for _, e := range s.Entries() {
77+
got.V = append(got.V, e.Vuln...)
78+
got.E = append(got.E, e.Enrichment...)
9079
}
9180
return nil
9281
})

‎libvuln/jsonblob/loader.go‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
package jsonblob
2+
3+
import (
4+
"encoding/json"
5+
"errors"
6+
"github.com/google/uuid"
7+
"io"
8+
9+
"github.com/quay/claircore"
10+
"github.com/quay/claircore/libvuln/driver"
11+
)
12+
13+
// NewLoader creates a new Loader from the provided [io.Reader].
14+
func NewLoader(r io.Reader) (*Loader, error) {
15+
l := Loader{
16+
dec: json.NewDecoder(r),
17+
cur: uuid.Nil,
18+
}
19+
return &l, nil
20+
}
21+
22+
// Loader is an iterator that returns a series of [Entry].
23+
//
24+
// Users should call [*Loader.Next] until it reports false, then check for
25+
// errors via [*Loader.Err].
26+
type Loader struct {
27+
err error
28+
e *Entry
29+
30+
dec *json.Decoder
31+
next *Entry
32+
de diskEntry
33+
cur uuid.UUID
34+
}
35+
36+
// Next reports whether there's an [Entry] to be processed.
37+
func (l *Loader) Next() bool {
38+
if l.err != nil {
39+
return false
40+
}
41+
42+
for l.err = l.dec.Decode(&l.de); l.err == nil; l.err = l.dec.Decode(&l.de) {
43+
id := l.de.Ref
44+
// If we just hit a new Entry, promote the current one.
45+
if id != l.cur {
46+
l.e = l.next
47+
l.next = &Entry{}
48+
l.next.Updater = l.de.Updater
49+
l.next.Fingerprint = l.de.Fingerprint
50+
l.next.Date = l.de.Date
51+
}
52+
switch l.de.Kind {
53+
case driver.VulnerabilityKind:
54+
vuln := claircore.Vulnerability{}
55+
if err := json.Unmarshal(l.de.Vuln.buf, &vuln); err != nil {
56+
l.err = err
57+
return false
58+
}
59+
l.next.Vuln = append(l.next.Vuln, &vuln)
60+
case driver.EnrichmentKind:
61+
en := driver.EnrichmentRecord{}
62+
if err := json.Unmarshal(l.de.Enrichment.buf, &en); err != nil {
63+
l.err = err
64+
return false
65+
}
66+
l.next.Enrichment = append(l.next.Enrichment, en)
67+
}
68+
// If this was an initial diskEntry, promote the ref.
69+
if id != l.cur {
70+
l.cur = id
71+
// If we have an Entry ready, report that.
72+
if l.e != nil {
73+
return true
74+
}
75+
}
76+
}
77+
l.e = l.next
78+
return true
79+
}
80+
81+
// Entry returns the latest loaded [Entry].
82+
func (l *Loader) Entry() *Entry {
83+
return l.e
84+
}
85+
86+
// Err is the latest encountered error.
87+
func (l *Loader) Err() error {
88+
// Don't report EOF as an error.
89+
if errors.Is(l.err, io.EOF) {
90+
return nil
91+
}
92+
return l.err
93+
}

‎libvuln/jsonblob/loader_test.go‎

Lines changed: 78 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,78 @@
1+
package jsonblob
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"io"
7+
"testing"
8+
9+
"github.com/google/go-cmp/cmp"
10+
"golang.org/x/sync/errgroup"
11+
12+
"github.com/quay/claircore"
13+
"github.com/quay/claircore/libvuln/driver"
14+
"github.com/quay/claircore/test"
15+
)
16+
17+
func TestLoader(t *testing.T) {
18+
ctx := context.Background()
19+
a, err := New()
20+
if err != nil {
21+
t.Fatal(err)
22+
}
23+
24+
var want, got struct {
25+
V []*claircore.Vulnerability
26+
E []driver.EnrichmentRecord
27+
}
28+
29+
want.V = test.GenUniqueVulnerabilities(10, "test")
30+
ref, err := a.UpdateVulnerabilities(ctx, "test", "", want.V)
31+
if err != nil {
32+
t.Error(err)
33+
}
34+
t.Logf("ref: %v", ref)
35+
36+
want.E = test.GenEnrichments(15)
37+
ref, err = a.UpdateEnrichments(ctx, "test", "", want.E)
38+
if err != nil {
39+
t.Error(err)
40+
}
41+
t.Logf("ref: %v", ref)
42+
43+
var buf bytes.Buffer
44+
defer func() {
45+
t.Logf("wrote:\n%s", buf.String())
46+
}()
47+
r, w := io.Pipe()
48+
eg, ctx := errgroup.WithContext(ctx)
49+
eg.Go(func() error { defer w.Close(); return a.Store(w) })
50+
eg.Go(func() error {
51+
l, err := NewLoader(io.TeeReader(r, &buf))
52+
if err != nil {
53+
return err
54+
}
55+
for l.Next() {
56+
e := l.Entry()
57+
if e.Vuln != nil && e.Enrichment != nil {
58+
t.Error("expecting entry to have either vulnerability or enrichment, got both")
59+
}
60+
if e.Vuln != nil {
61+
got.V = append(got.V, l.Entry().Vuln...)
62+
}
63+
if e.Enrichment != nil {
64+
got.E = append(got.E, l.Entry().Enrichment...)
65+
}
66+
}
67+
if err := l.Err(); err != nil {
68+
return err
69+
}
70+
return nil
71+
})
72+
if err := eg.Wait(); err != nil {
73+
t.Error(err)
74+
}
75+
if !cmp.Equal(got, want) {
76+
t.Error(cmp.Diff(got, want))
77+
}
78+
}

‎libvuln/updates.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ func OfflineImport(ctx context.Context, pool *pgxpool.Pool, in io.Reader) error
2626
ctx = zlog.ContextWithValues(ctx, "component", "libvuln/OfflineImporter")
2727

2828
s := postgres.NewMatcherStore(pool)
29-
l, err := jsonblob.Load(ctx, in)
29+
l, err := jsonblob.NewLoader(in)
3030
if err != nil {
3131
return err
3232
}

0 commit comments

Comments
 (0)