|
9 | 9 |
|
10 | 10 | package org.elasticsearch.repositories.gcs; |
11 | 11 |
|
12 | | -import com.google.api.client.http.GenericUrl; |
13 | | -import com.google.api.client.http.HttpRequest; |
14 | 12 | import com.google.api.client.http.HttpResponse; |
15 | 13 | import com.google.api.client.http.HttpResponseInterceptor; |
16 | 14 |
|
17 | 15 | import org.elasticsearch.common.blobstore.OperationPurpose; |
| 16 | +import org.elasticsearch.logging.LogManager; |
| 17 | +import org.elasticsearch.logging.Logger; |
18 | 18 |
|
19 | | -import java.util.List; |
20 | | -import java.util.Locale; |
21 | | -import java.util.function.Consumer; |
22 | | -import java.util.function.Function; |
23 | 19 | import java.util.regex.Pattern; |
24 | 20 |
|
25 | | -import static java.lang.String.format; |
| 21 | +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageOperationsStats.Operation; |
| 22 | +import static org.elasticsearch.repositories.gcs.GoogleCloudStorageOperationsStats.StatsTracker; |
26 | 23 |
|
27 | 24 | final class GoogleCloudStorageHttpStatsCollector implements HttpResponseInterceptor { |
28 | | - // The specification for the current API (v1) endpoints can be found at: |
29 | | - // https://cloud.google.com/storage/docs/json_api/v1 |
30 | | - private static final List<Function<String, HttpRequestTracker>> trackerFactories = List.of( |
31 | | - (bucket) -> HttpRequestTracker.get( |
32 | | - format(Locale.ROOT, "/download/storage/v1/b/%s/o/.+", bucket), |
33 | | - GoogleCloudStorageOperationsStats::trackGetOperation |
34 | | - ), |
35 | 25 |
|
36 | | - (bucket) -> HttpRequestTracker.get( |
37 | | - format(Locale.ROOT, "/storage/v1/b/%s/o/.+", bucket), |
38 | | - GoogleCloudStorageOperationsStats::trackGetOperation |
39 | | - ), |
| 26 | + private static final Logger logger = LogManager.getLogger("GcpHttpStats"); |
40 | 27 |
|
41 | | - (bucket) -> HttpRequestTracker.get( |
42 | | - format(Locale.ROOT, "/storage/v1/b/%s/o", bucket), |
43 | | - GoogleCloudStorageOperationsStats::trackListOperation |
44 | | - ) |
45 | | - ); |
| 28 | + private final StatsTracker stats; |
| 29 | + private final OperationPurpose purpose; |
| 30 | + private final Pattern getObjPattern; |
| 31 | + private final Pattern insertObjPattern; |
| 32 | + private final Pattern listObjPattern; |
46 | 33 |
|
47 | | - private final GoogleCloudStorageOperationsStats gcsOperationStats; |
48 | | - private final OperationPurpose operationPurpose; |
49 | | - private final List<HttpRequestTracker> trackers; |
| 34 | + GoogleCloudStorageHttpStatsCollector(final GoogleCloudStorageOperationsStats stats, OperationPurpose purpose) { |
| 35 | + this.stats = stats.tracker(); |
| 36 | + this.purpose = purpose; |
| 37 | + var bucket = stats.bucketName(); |
50 | 38 |
|
51 | | - GoogleCloudStorageHttpStatsCollector(final GoogleCloudStorageOperationsStats gcsOperationStats, OperationPurpose operationPurpose) { |
52 | | - this.gcsOperationStats = gcsOperationStats; |
53 | | - this.operationPurpose = operationPurpose; |
54 | | - this.trackers = trackerFactories.stream() |
55 | | - .map(trackerFactory -> trackerFactory.apply(gcsOperationStats.getTrackedBucket())) |
56 | | - .toList(); |
| 39 | + // The specification for the current API (v1) endpoints can be found at: |
| 40 | + // https://cloud.google.com/storage/docs/json_api/v1 |
| 41 | + this.getObjPattern = Pattern.compile("(/download)?/storage/v1/b/" + bucket + "/o/.+"); |
| 42 | + this.insertObjPattern = Pattern.compile("(/upload)?/storage/v1/b/" + bucket + "/o"); |
| 43 | + this.listObjPattern = Pattern.compile("/storage/v1/b/" + bucket + "/o"); |
57 | 44 | } |
58 | 45 |
|
59 | | - @Override |
60 | | - public void interceptResponse(final HttpResponse response) { |
61 | | - // TODO keep track of unsuccessful requests in different entries |
62 | | - if (response.isSuccessStatusCode() == false) return; |
63 | | - |
64 | | - final HttpRequest request = response.getRequest(); |
65 | | - for (HttpRequestTracker tracker : trackers) { |
66 | | - if (tracker.track(request, gcsOperationStats)) { |
67 | | - return; |
68 | | - } |
69 | | - } |
| 46 | + private void trackRequest(Operation operation) { |
| 47 | + stats.trackRequest(purpose, operation); |
70 | 48 | } |
71 | 49 |
|
72 | | - /** |
73 | | - * Http request tracker that allows to track certain HTTP requests based on the following criteria: |
74 | | - * <ul> |
75 | | - * <li>The HTTP request method</li> |
76 | | - * <li>An URI path regex expression</li> |
77 | | - * </ul> |
78 | | - * |
79 | | - * The requests that match the previous criteria are tracked using the {@code statsTracker} function. |
80 | | - */ |
81 | | - private static final class HttpRequestTracker { |
82 | | - private final String method; |
83 | | - private final Pattern pathPattern; |
84 | | - private final Consumer<GoogleCloudStorageOperationsStats> statsTracker; |
85 | | - |
86 | | - private HttpRequestTracker( |
87 | | - final String method, |
88 | | - final String pathPattern, |
89 | | - final Consumer<GoogleCloudStorageOperationsStats> statsTracker |
90 | | - ) { |
91 | | - this.method = method; |
92 | | - this.pathPattern = Pattern.compile(pathPattern); |
93 | | - this.statsTracker = statsTracker; |
94 | | - } |
95 | | - |
96 | | - private static HttpRequestTracker get(final String pathPattern, final Consumer<GoogleCloudStorageOperationsStats> statsConsumer) { |
97 | | - return new HttpRequestTracker("GET", pathPattern, statsConsumer); |
98 | | - } |
99 | | - |
100 | | - /** |
101 | | - * Tracks the provided http request if it matches the criteria defined by this tracker. |
102 | | - * |
103 | | - * @param httpRequest the http request to be tracked |
104 | | - * @param stats the operation tracker |
105 | | - * |
106 | | - * @return {@code true} if the http request was tracked, {@code false} otherwise. |
107 | | - */ |
108 | | - private boolean track(final HttpRequest httpRequest, final GoogleCloudStorageOperationsStats stats) { |
109 | | - if (matchesCriteria(httpRequest) == false) return false; |
| 50 | + private void trackRequestAndOperation(Operation operation) { |
| 51 | + stats.trackOperationAndRequest(purpose, operation); |
| 52 | + } |
110 | 53 |
|
111 | | - statsTracker.accept(stats); |
112 | | - return true; |
| 54 | + @Override |
| 55 | + public void interceptResponse(final HttpResponse response) { |
| 56 | + var respCode = response.getStatusCode(); |
| 57 | + // Some of the intermediate and error codes are still counted as "good" requests |
| 58 | + if (((respCode >= 200 && respCode < 300) || respCode == 308 || respCode == 404) == false) { |
| 59 | + return; |
113 | 60 | } |
114 | | - |
115 | | - private boolean matchesCriteria(final HttpRequest httpRequest) { |
116 | | - return method.equalsIgnoreCase(httpRequest.getRequestMethod()) && pathMatches(httpRequest.getUrl()); |
| 61 | + var request = response.getRequest(); |
| 62 | + |
| 63 | + var path = request.getUrl().getRawPath(); |
| 64 | + var ignored = false; |
| 65 | + switch (request.getRequestMethod()) { |
| 66 | + case "GET" -> { |
| 67 | + // https://cloud.google.com/storage/docs/json_api/v1/objects/get |
| 68 | + if (getObjPattern.matcher(path).matches()) { |
| 69 | + trackRequestAndOperation(Operation.GET_OBJECT); |
| 70 | + } else if (listObjPattern.matcher(path).matches()) { |
| 71 | + trackRequestAndOperation(Operation.LIST_OBJECTS); |
| 72 | + } else { |
| 73 | + ignored = true; |
| 74 | + } |
| 75 | + } |
| 76 | + case "POST", "PUT" -> { |
| 77 | + // https://cloud.google.com/storage/docs/json_api/v1/objects/insert |
| 78 | + if (insertObjPattern.matcher(path).matches()) { |
| 79 | + trackRequest(Operation.INSERT_OBJECT); |
| 80 | + } else { |
| 81 | + ignored = true; |
| 82 | + } |
| 83 | + } |
| 84 | + default -> ignored = true; |
117 | 85 | } |
118 | | - |
119 | | - private boolean pathMatches(final GenericUrl url) { |
120 | | - return pathPattern.matcher(url.getRawPath()).matches(); |
| 86 | + if (ignored) { |
| 87 | + logger.debug( |
| 88 | + "not handling request:{} {} response:{} {}", |
| 89 | + request.getRequestMethod(), |
| 90 | + path, |
| 91 | + response.getStatusCode(), |
| 92 | + response.getStatusMessage() |
| 93 | + ); |
121 | 94 | } |
122 | 95 | } |
123 | 96 | } |
0 commit comments