Skip to content
This repository was archived by the owner on Oct 2, 2025. It is now read-only.

Commit 178156d

Browse files
cfiebercfieber
committed
instrumented StorageServiceSupport
increased health check timeout to 90s and fail if a poll interval larger than that is requested adds storageServiceSupport.cacheAge gauge and storageServiceSupport.cacheRefreshTime timers
1 parent ffe26c2 commit 178156d

File tree

17 files changed

+112
-68
lines changed

17 files changed

+112
-68
lines changed

‎front50-core/src/main/groovy/com/netflix/spinnaker/front50/model/StorageServiceSupport.java‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
*/
1616
package com.netflix.spinnaker.front50.model;
1717

18+
import com.netflix.spectator.api.Registry;
1819
import com.netflix.spinnaker.front50.exception.NotFoundException;
1920
import com.netflix.spinnaker.front50.support.ClosureHelper;
2021
import com.netflix.spinnaker.hystrix.SimpleHystrixCommand;
@@ -27,32 +28,43 @@
2728
import javax.annotation.PostConstruct;
2829
import java.lang.Long;
2930
import java.util.*;
31+
import java.util.concurrent.atomic.AtomicLong;
3032
import java.util.concurrent.atomic.AtomicReference;
3133
import java.util.concurrent.TimeUnit;
3234
import java.util.function.Function;
3335
import java.util.stream.Collectors;
3436

3537

3638
public abstract class StorageServiceSupport<T extends Timestamped> {
37-
public static long HEALTH_MILLIS = 45000;
39+
private static final long HEALTH_MILLIS = TimeUnit.SECONDS.toMillis(90);
3840
private final Logger log = LoggerFactory.getLogger(getClass());
3941
protected final AtomicReference<Set<T>> allItemsCache = new AtomicReference<>();
4042

4143
private final ObjectType objectType;
4244
private final StorageService service;
4345
private final Scheduler scheduler;
4446
private final int refreshIntervalMs;
47+
private final Registry registry;
4548

46-
private long lastRefreshedTime;
49+
private final AtomicLong lastRefreshedTime = new AtomicLong();
4750

4851
public StorageServiceSupport(ObjectType objectType,
4952
StorageService service,
5053
Scheduler scheduler,
51-
int refreshIntervalMs) {
54+
int refreshIntervalMs,
55+
Registry registry) {
5256
this.objectType = objectType;
5357
this.service = service;
5458
this.scheduler = scheduler;
5559
this.refreshIntervalMs = refreshIntervalMs;
60+
if (refreshIntervalMs >= HEALTH_MILLIS) {
61+
throw new IllegalArgumentException("Cache refresh time must be more frequent than cache health timeout");
62+
}
63+
this.registry = registry;
64+
registry.gauge(
65+
registry.createId("storageServiceSupport.cacheAge", "objectType", objectType.name()),
66+
lastRefreshedTime,
67+
(lrt) -> Long.valueOf(System.currentTimeMillis() - lrt.get()).doubleValue());
5668
}
5769

5870
@PostConstruct
@@ -80,7 +92,7 @@ void startRefresh() {
8092

8193
public Collection<T> all() {
8294
long lastModified = readLastModified();
83-
if (lastModified > lastRefreshedTime || allItemsCache.get() == null) {
95+
if (lastModified > lastRefreshedTime.get() || allItemsCache.get() == null) {
8496
// only refresh if there was a modification since our last refresh cycle
8597
log.debug("all() forcing refresh");
8698
refresh();
@@ -101,7 +113,7 @@ public Collection<T> history(String id, int maxResults) {
101113
* @return Healthy if refreshed in the past HEALTH_MILLIS
102114
*/
103115
public boolean isHealthy() {
104-
return (System.currentTimeMillis() - lastRefreshedTime) < HEALTH_MILLIS && allItemsCache.get() != null;
116+
return (System.currentTimeMillis() - lastRefreshedTime.get()) < HEALTH_MILLIS && allItemsCache.get() != null;
105117
}
106118

107119
public T findById(String id) throws NotFoundException {
@@ -147,11 +159,13 @@ public void bulkImport(Collection<T> items) {
147159
* Update local cache with any recently modified items.
148160
*/
149161
protected void refresh() {
150-
long startTime = System.currentTimeMillis();
162+
long startTime = System.nanoTime();
151163
allItemsCache.set(fetchAllItems(allItemsCache.get()));
152-
long endTime = System.currentTimeMillis();
164+
long elapsed = System.nanoTime() - startTime;
165+
166+
registry.timer("storageServiceSupport.cacheRefreshTime", "objectType", objectType.name()).record(elapsed, TimeUnit.NANOSECONDS);
153167

154-
log.debug("Refreshed (" + (endTime - startTime) + "ms)");
168+
log.debug("Refreshed (" + TimeUnit.NANOSECONDS.toMillis(elapsed) + "ms)");
155169
}
156170

157171
private String buildObjectKey(T item) {
@@ -226,7 +240,7 @@ private Set<T> fetchAllItems(Set<T> existingItems) {
226240
});
227241

228242
Set<T> result = resultMap.values().stream().collect(Collectors.toSet());
229-
this.lastRefreshedTime = refreshTime;
243+
this.lastRefreshedTime.set(refreshTime);
230244

231245
int result_size = result.size();
232246
if (existingSize != result_size) {

‎front50-core/src/main/groovy/com/netflix/spinnaker/front50/model/application/DefaultApplicationDAO.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package com.netflix.spinnaker.front50.model.application;
1818

19+
import com.netflix.spectator.api.Registry;
1920
import com.netflix.spinnaker.front50.exception.NotFoundException;
2021
import com.netflix.spinnaker.front50.model.ObjectType;
2122
import com.netflix.spinnaker.front50.model.StorageService;
@@ -28,8 +29,9 @@
2829
public class DefaultApplicationDAO extends StorageServiceSupport<Application> implements ApplicationDAO {
2930
public DefaultApplicationDAO(StorageService service,
3031
Scheduler scheduler,
31-
int refreshIntervalMs) {
32-
super(ObjectType.APPLICATION, service, scheduler, refreshIntervalMs);
32+
int refreshIntervalMs,
33+
Registry registry) {
34+
super(ObjectType.APPLICATION, service, scheduler, refreshIntervalMs, registry);
3335
}
3436

3537
@Override

‎front50-core/src/main/groovy/com/netflix/spinnaker/front50/model/application/DefaultApplicationPermissionDAO.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package com.netflix.spinnaker.front50.model.application;
1818

19+
import com.netflix.spectator.api.Registry;
1920
import com.netflix.spinnaker.front50.model.ObjectType;
2021
import com.netflix.spinnaker.front50.model.StorageService;
2122
import com.netflix.spinnaker.front50.model.StorageServiceSupport;
@@ -24,8 +25,9 @@
2425
public class DefaultApplicationPermissionDAO extends StorageServiceSupport<Application.Permission> implements ApplicationPermissionDAO {
2526
public DefaultApplicationPermissionDAO(StorageService service,
2627
Scheduler scheduler,
27-
int refreshIntervalMs) {
28-
super(ObjectType.APPLICATION_PERMISSION, service, scheduler, refreshIntervalMs);
28+
int refreshIntervalMs,
29+
Registry registry) {
30+
super(ObjectType.APPLICATION_PERMISSION, service, scheduler, refreshIntervalMs, registry);
2931
}
3032

3133
@Override

‎front50-core/src/main/groovy/com/netflix/spinnaker/front50/model/notification/DefaultNotificationDAO.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package com.netflix.spinnaker.front50.model.notification;
1818

19+
import com.netflix.spectator.api.Registry;
1920
import com.netflix.spinnaker.front50.exception.NotFoundException;
2021
import com.netflix.spinnaker.front50.model.ObjectType;
2122
import com.netflix.spinnaker.front50.model.StorageService;
@@ -25,8 +26,9 @@
2526
public class DefaultNotificationDAO extends StorageServiceSupport<Notification> implements NotificationDAO {
2627
public DefaultNotificationDAO(StorageService service,
2728
Scheduler scheduler,
28-
int refreshIntervalMs) {
29-
super(ObjectType.NOTIFICATION, service, scheduler, refreshIntervalMs);
29+
int refreshIntervalMs,
30+
Registry registry) {
31+
super(ObjectType.NOTIFICATION, service, scheduler, refreshIntervalMs, registry);
3032
}
3133

3234
@Override

‎front50-core/src/main/groovy/com/netflix/spinnaker/front50/model/pipeline/DefaultPipelineDAO.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package com.netflix.spinnaker.front50.model.pipeline;
1818

19+
import com.netflix.spectator.api.Registry;
1920
import com.netflix.spinnaker.front50.exception.NotFoundException;
2021
import com.netflix.spinnaker.front50.model.ObjectType;
2122
import com.netflix.spinnaker.front50.model.StorageService;
@@ -30,8 +31,9 @@
3031
public class DefaultPipelineDAO extends StorageServiceSupport<Pipeline> implements PipelineDAO {
3132
public DefaultPipelineDAO(StorageService service,
3233
Scheduler scheduler,
33-
int refreshIntervalMs) {
34-
super(ObjectType.PIPELINE, service, scheduler, refreshIntervalMs);
34+
int refreshIntervalMs,
35+
Registry registry) {
36+
super(ObjectType.PIPELINE, service, scheduler, refreshIntervalMs, registry);
3537
}
3638

3739
@Override

‎front50-core/src/main/groovy/com/netflix/spinnaker/front50/model/pipeline/DefaultPipelineStrategyDAO.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package com.netflix.spinnaker.front50.model.pipeline;
1818

19+
import com.netflix.spectator.api.Registry;
1920
import com.netflix.spinnaker.front50.exception.NotFoundException;
2021
import com.netflix.spinnaker.front50.model.ObjectType;
2122
import com.netflix.spinnaker.front50.model.StorageService;
@@ -28,9 +29,10 @@
2829

2930
public class DefaultPipelineStrategyDAO extends StorageServiceSupport<Pipeline> implements PipelineStrategyDAO {
3031
public DefaultPipelineStrategyDAO(StorageService service,
31-
Scheduler scheduler,
32-
int refreshIntervalMs) {
33-
super(ObjectType.STRATEGY, service, scheduler, refreshIntervalMs);
32+
Scheduler scheduler,
33+
int refreshIntervalMs,
34+
Registry registry) {
35+
super(ObjectType.STRATEGY, service, scheduler, refreshIntervalMs, registry);
3436
}
3537

3638
@Override

‎front50-core/src/main/groovy/com/netflix/spinnaker/front50/model/project/DefaultProjectDAO.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package com.netflix.spinnaker.front50.model.project;
1818

19+
import com.netflix.spectator.api.Registry;
1920
import com.netflix.spinnaker.front50.exception.NotFoundException;
2021
import com.netflix.spinnaker.front50.model.ObjectType;
2122
import com.netflix.spinnaker.front50.model.StorageService;
@@ -26,9 +27,10 @@
2627

2728
public class DefaultProjectDAO extends StorageServiceSupport<Project> implements ProjectDAO {
2829
public DefaultProjectDAO(StorageService service,
29-
Scheduler scheduler,
30-
int refreshIntervalMs) {
31-
super(ObjectType.PROJECT, service, scheduler, refreshIntervalMs);
30+
Scheduler scheduler,
31+
int refreshIntervalMs,
32+
Registry registry) {
33+
super(ObjectType.PROJECT, service, scheduler, refreshIntervalMs, registry);
3234
}
3335

3436
@Override

‎front50-core/src/main/groovy/com/netflix/spinnaker/front50/model/serviceaccount/DefaultServiceAccountDAO.java‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,16 +16,18 @@
1616

1717
package com.netflix.spinnaker.front50.model.serviceaccount;
1818

19+
import com.netflix.spectator.api.Registry;
1920
import com.netflix.spinnaker.front50.model.ObjectType;
2021
import com.netflix.spinnaker.front50.model.StorageService;
2122
import com.netflix.spinnaker.front50.model.StorageServiceSupport;
2223
import rx.Scheduler;
2324

2425
public class DefaultServiceAccountDAO extends StorageServiceSupport<ServiceAccount> implements ServiceAccountDAO {
2526
public DefaultServiceAccountDAO(StorageService service,
26-
Scheduler scheduler,
27-
int refreshIntervalMs) {
28-
super(ObjectType.SERVICE_ACCOUNT, service, scheduler, refreshIntervalMs);
27+
Scheduler scheduler,
28+
int refreshIntervalMs,
29+
Registry registry) {
30+
super(ObjectType.SERVICE_ACCOUNT, service, scheduler, refreshIntervalMs, registry);
2931
}
3032

3133
@Override

‎front50-core/src/main/groovy/com/netflix/spinnaker/front50/model/snapshot/DefaultSnapshotDAO.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package com.netflix.spinnaker.front50.model.snapshot;
1818

19+
import com.netflix.spectator.api.Registry;
1920
import com.netflix.spinnaker.front50.model.ObjectType;
2021
import com.netflix.spinnaker.front50.model.StorageService;
2122
import com.netflix.spinnaker.front50.model.StorageServiceSupport;
@@ -27,8 +28,9 @@
2728
public class DefaultSnapshotDAO extends StorageServiceSupport<Snapshot> implements SnapshotDAO {
2829
public DefaultSnapshotDAO(StorageService service,
2930
Scheduler scheduler,
30-
int refreshIntervalMs) {
31-
super(ObjectType.SNAPSHOT, service, scheduler, refreshIntervalMs);
31+
int refreshIntervalMs,
32+
Registry registry) {
33+
super(ObjectType.SNAPSHOT, service, scheduler, refreshIntervalMs, registry);
3234
}
3335

3436
@Override

‎front50-core/src/main/groovy/com/netflix/spinnaker/front50/model/tag/DefaultEntityTagsDAO.java‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616

1717
package com.netflix.spinnaker.front50.model.tag;
1818

19+
import com.netflix.spectator.api.NoopRegistry;
20+
import com.netflix.spectator.api.Registry;
1921
import com.netflix.spinnaker.front50.model.ObjectType;
2022
import com.netflix.spinnaker.front50.model.StorageService;
2123
import com.netflix.spinnaker.front50.model.StorageServiceSupport;
@@ -27,9 +29,9 @@
2729

2830
public class DefaultEntityTagsDAO extends StorageServiceSupport<EntityTags> implements EntityTagsDAO {
2931
public DefaultEntityTagsDAO(StorageService service,
30-
Scheduler scheduler,
31-
int refreshIntervalMs) {
32-
super(ObjectType.ENTITY_TAGS, service, scheduler, refreshIntervalMs);
32+
Scheduler scheduler,
33+
int refreshIntervalMs) {
34+
super(ObjectType.ENTITY_TAGS, service, scheduler, refreshIntervalMs, new NoopRegistry());
3335
}
3436

3537
@Override
@@ -58,4 +60,9 @@ public Collection<EntityTags> all() {
5860
protected void refresh() {
5961
// avoid loading all tagged entities into memory
6062
}
63+
64+
@Override
65+
public boolean isHealthy() {
66+
return true;
67+
}
6168
}

0 commit comments

Comments
 (0)