From 89729a96492673de76642e576f4f3361731d03ce Mon Sep 17 00:00:00 2001 From: tschoemaker Date: Fri, 3 May 2024 16:48:10 +0200 Subject: [PATCH 1/2] with errors --- .../GoogleDtpInternalMetricRecorder.java | 649 +++++++++--------- .../AzureDtpInternalMetricRecorder.java | 253 ++++--- .../launcher/DtpInternalMetricRecorder.java | 100 ++- .../api/launcher/ServiceResult.java | 39 ++ .../LoggingDtpInternalMetricRecorder.java | 275 ++++---- .../transfer/CallableExporter.java | 66 +- .../transfer/CallableImporter.java | 92 +-- ...PortabilityAbstractInMemoryDataCopier.java | 431 ++++++------ 8 files changed, 969 insertions(+), 936 deletions(-) create mode 100644 portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/ServiceResult.java diff --git a/extensions/cloud/portability-cloud-google/src/main/java/org/datatransferproject/cloud/google/GoogleDtpInternalMetricRecorder.java b/extensions/cloud/portability-cloud-google/src/main/java/org/datatransferproject/cloud/google/GoogleDtpInternalMetricRecorder.java index f3e6649ad..481ccd05d 100644 --- a/extensions/cloud/portability-cloud-google/src/main/java/org/datatransferproject/cloud/google/GoogleDtpInternalMetricRecorder.java +++ b/extensions/cloud/portability-cloud-google/src/main/java/org/datatransferproject/cloud/google/GoogleDtpInternalMetricRecorder.java @@ -20,356 +20,339 @@ import io.opencensus.common.Scope; import io.opencensus.exporter.stats.stackdriver.StackdriverStatsConfiguration; import io.opencensus.exporter.stats.stackdriver.StackdriverStatsExporter; -import io.opencensus.stats.Aggregation; -import io.opencensus.stats.Measure; -import io.opencensus.stats.Stats; -import io.opencensus.stats.StatsRecorder; -import io.opencensus.stats.View; -import io.opencensus.stats.ViewManager; -import io.opencensus.tags.TagContext; -import io.opencensus.tags.TagKey; -import io.opencensus.tags.TagMetadata; -import io.opencensus.tags.TagValue; -import io.opencensus.tags.Tagger; -import io.opencensus.tags.Tags; +import io.opencensus.stats.*; +import io.opencensus.tags.*; import org.datatransferproject.api.launcher.DtpInternalMetricRecorder; +import org.datatransferproject.api.launcher.ServiceResult; +import org.datatransferproject.types.common.models.DataVertical; import java.io.IOException; import java.time.Duration; -import org.datatransferproject.types.common.models.DataVertical; /** * A {@link DtpInternalMetricRecorder} that writes metrics to Stackdriver. - * **/ + **/ class GoogleDtpInternalMetricRecorder implements DtpInternalMetricRecorder { - private static GoogleDtpInternalMetricRecorder INSTANCE = null; - - private static final int EXPORT_INTERVAL_SECONDS = 60; - private static final String NAME_BASE = "dtp/"; - private static final StatsRecorder STATS_RECORDER = Stats.getStatsRecorder(); - private static final Tagger tagger = Tags.getTagger(); - - private static final TagKey KEY_DATA_TYPE = TagKey.create("data_type"); - private static final TagKey KEY_IMPORT_SERVICE = TagKey.create("import_service"); - private static final TagKey KEY_EXPORT_SERVICE = TagKey.create("export_service"); - private static final TagKey KEY_SUCCESS = TagKey.create("success"); - - private static final TagKey KEY_GENERIC_SERVICE = TagKey.create("generic_service"); - private static final TagKey KEY_GENERIC_TAG = TagKey.create("generic_tag"); - private static final TagKey KEY_GENERIC_BOOL = TagKey.create("generic_bool"); - - private static final TagMetadata TAG_METADATA = - TagMetadata.create(TagMetadata.TagTtl.UNLIMITED_PROPAGATION); - - private static final Measure.MeasureLong JOB_STARTED = Measure.MeasureLong.create( - "job_start", - "Number of jobs that were started", - "count"); - - private static final Measure.MeasureLong JOB_FINISHED = Measure.MeasureLong.create( - "job_finished", - "Number of jobs that finished", - "count"); - - private static final Measure.MeasureLong JOB_FINISHED_DURATION = Measure.MeasureLong.create( - "job_finished_duration", - "Duration of a job in MS", - "ms"); - - private static final Measure.MeasureLong EXPORT_PAGE_ATTEMPT = Measure.MeasureLong.create( - "export_page_attempt", - "A single export attempt", - "count"); - - private static final Measure.MeasureLong EXPORT_PAGE_ATTEMPT_DURATION = - Measure.MeasureLong.create( - "export_page_attempt_duration", - "Duration of a single export attempt in MS", - "ms"); - - private static final Measure.MeasureLong EXPORT_PAGE = Measure.MeasureLong.create( - "export_page", - "An export attempt including all retries", - "count"); - - private static final Measure.MeasureLong EXPORT_PAGE_DURATION = - Measure.MeasureLong.create( - "export_page_duration", - "Duration of an export page including retries in MS", - "ms"); - - private static final Measure.MeasureLong IMPORT_PAGE_ATTEMPT = Measure.MeasureLong.create( - "import_page_attempt", - "A single import attempt", - "count"); - - private static final Measure.MeasureLong IMPORT_PAGE_ATTEMPT_DURATION = - Measure.MeasureLong.create( - "import_page_attempt_duration", - "Duration of a single import attempt in MS", - "ms"); - - private static final Measure.MeasureLong IMPORT_PAGE = Measure.MeasureLong.create( - "import_page", - "An import attempt including all retries", - "count"); - - private static final Measure.MeasureLong IMPORT_PAGE_DURATION = - Measure.MeasureLong.create( - "import_page_duration", - "Duration of an import page including retries in MS", - "ms"); - - private static final Measure.MeasureLong GENERIC_COUNT = Measure.MeasureLong.create( - "generic_count", - "A generic counter that services can use to hold arbitrary metrics", - "count"); - - private static final Measure.MeasureLong GENERIC_DURATION = Measure.MeasureLong.create( - "generic_duration", - "A generic counter that services can use to hold arbitrary duration metrics", - "ms"); - - private static final Measure.MeasureLong GENERIC_BOOLEAN = Measure.MeasureLong.create( - "generic_boolean", - "A generic counter that services can use to hold arbitrary boolean metrics", - "count"); - - private final ViewManager viewManager; - - // This is needed because Stackdriver can only be initialized once and the - // GoogleDtpInternalMetricExtension is call more than once by the framework code. - static synchronized GoogleDtpInternalMetricRecorder getInstance() throws IOException { - if (INSTANCE == null) { - INSTANCE = new GoogleDtpInternalMetricRecorder( - GoogleCredentials.getApplicationDefault(), - GoogleCloudUtils.getProjectId()); + private static GoogleDtpInternalMetricRecorder INSTANCE = null; + + private static final int EXPORT_INTERVAL_SECONDS = 60; + private static final String NAME_BASE = "dtp/"; + private static final StatsRecorder STATS_RECORDER = Stats.getStatsRecorder(); + private static final Tagger tagger = Tags.getTagger(); + + private static final TagKey KEY_DATA_TYPE = TagKey.create("data_type"); + private static final TagKey KEY_IMPORT_SERVICE = TagKey.create("import_service"); + private static final TagKey KEY_EXPORT_SERVICE = TagKey.create("export_service"); + private static final TagKey KEY_SUCCESS = TagKey.create("success"); + + private static final TagKey KEY_GENERIC_SERVICE = TagKey.create("generic_service"); + private static final TagKey KEY_GENERIC_TAG = TagKey.create("generic_tag"); + private static final TagKey KEY_GENERIC_BOOL = TagKey.create("generic_bool"); + + private static final TagMetadata TAG_METADATA = + TagMetadata.create(TagMetadata.TagTtl.UNLIMITED_PROPAGATION); + + private static final Measure.MeasureLong JOB_STARTED = Measure.MeasureLong.create( + "job_start", + "Number of jobs that were started", + "count"); + + private static final Measure.MeasureLong JOB_FINISHED = Measure.MeasureLong.create( + "job_finished", + "Number of jobs that finished", + "count"); + + private static final Measure.MeasureLong JOB_FINISHED_DURATION = Measure.MeasureLong.create( + "job_finished_duration", + "Duration of a job in MS", + "ms"); + + private static final Measure.MeasureLong EXPORT_PAGE_ATTEMPT = Measure.MeasureLong.create( + "export_page_attempt", + "A single export attempt", + "count"); + + private static final Measure.MeasureLong EXPORT_PAGE_ATTEMPT_DURATION = + Measure.MeasureLong.create( + "export_page_attempt_duration", + "Duration of a single export attempt in MS", + "ms"); + + private static final Measure.MeasureLong EXPORT_PAGE = Measure.MeasureLong.create( + "export_page", + "An export attempt including all retries", + "count"); + + private static final Measure.MeasureLong EXPORT_PAGE_DURATION = + Measure.MeasureLong.create( + "export_page_duration", + "Duration of an export page including retries in MS", + "ms"); + + private static final Measure.MeasureLong IMPORT_PAGE_ATTEMPT = Measure.MeasureLong.create( + "import_page_attempt", + "A single import attempt", + "count"); + + private static final Measure.MeasureLong IMPORT_PAGE_ATTEMPT_DURATION = + Measure.MeasureLong.create( + "import_page_attempt_duration", + "Duration of a single import attempt in MS", + "ms"); + + private static final Measure.MeasureLong IMPORT_PAGE = Measure.MeasureLong.create( + "import_page", + "An import attempt including all retries", + "count"); + + private static final Measure.MeasureLong IMPORT_PAGE_DURATION = + Measure.MeasureLong.create( + "import_page_duration", + "Duration of an import page including retries in MS", + "ms"); + + private static final Measure.MeasureLong GENERIC_COUNT = Measure.MeasureLong.create( + "generic_count", + "A generic counter that services can use to hold arbitrary metrics", + "count"); + + private static final Measure.MeasureLong GENERIC_DURATION = Measure.MeasureLong.create( + "generic_duration", + "A generic counter that services can use to hold arbitrary duration metrics", + "ms"); + + private static final Measure.MeasureLong GENERIC_BOOLEAN = Measure.MeasureLong.create( + "generic_boolean", + "A generic counter that services can use to hold arbitrary boolean metrics", + "count"); + + private final ViewManager viewManager; + + // This is needed because Stackdriver can only be initialized once and the + // GoogleDtpInternalMetricExtension is call more than once by the framework code. + static synchronized GoogleDtpInternalMetricRecorder getInstance() throws IOException { + if (INSTANCE == null) { + INSTANCE = new GoogleDtpInternalMetricRecorder( + GoogleCredentials.getApplicationDefault(), + GoogleCloudUtils.getProjectId()); + } + + return INSTANCE; + } + + private GoogleDtpInternalMetricRecorder(GoogleCredentials credentials, String projectId) + throws IOException { + // Enable OpenCensus exporters to export metrics to Stackdriver Monitoring. + // Exporters use Application Default Credentials to authenticate. + // See https://developers.google.com/identity/protocols/application-default-credentials + // for more details. + StackdriverStatsConfiguration configuration = StackdriverStatsConfiguration.builder() + .setCredentials(credentials) + .setProjectId(projectId) + .setExportInterval(io.opencensus.common.Duration.create(EXPORT_INTERVAL_SECONDS, 0)) + .build(); + StackdriverStatsExporter.createAndRegister(configuration); + this.viewManager = Stats.getViewManager(); + setupViews(); + } + + private void setupViews() { + setupView(JOB_STARTED, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_IMPORT_SERVICE); + + setupView(JOB_FINISHED, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_IMPORT_SERVICE, KEY_SUCCESS); + setupView( + JOB_FINISHED_DURATION, + KEY_DATA_TYPE, + KEY_EXPORT_SERVICE, + KEY_IMPORT_SERVICE, + KEY_SUCCESS); + + setupView(EXPORT_PAGE_ATTEMPT, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); + setupView(EXPORT_PAGE_ATTEMPT_DURATION, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); + + setupView(EXPORT_PAGE, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); + setupView(EXPORT_PAGE_DURATION, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); + + setupView(IMPORT_PAGE_ATTEMPT, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); + setupView(IMPORT_PAGE_ATTEMPT_DURATION, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); + + setupView(IMPORT_PAGE, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); + setupView(IMPORT_PAGE_DURATION, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); + + setupView(GENERIC_COUNT, KEY_DATA_TYPE, KEY_GENERIC_SERVICE, KEY_GENERIC_TAG); + setupView( + GENERIC_BOOLEAN, + KEY_DATA_TYPE, + KEY_GENERIC_SERVICE, + KEY_GENERIC_TAG, + KEY_GENERIC_BOOL); + setupView(GENERIC_DURATION, KEY_DATA_TYPE, KEY_GENERIC_SERVICE, KEY_GENERIC_TAG); + } + + private void setupView(Measure measure, TagKey... keys) { + // Register the view. It is imperative that this step exists, + // otherwise recorded metrics will be dropped and never exported. + View view = View.create( + View.Name.create(NAME_BASE + measure.getName()), + measure.getDescription(), + measure, + Aggregation.Count.create(), + ImmutableList.copyOf(keys)); + + viewManager.registerView(view); + } + + @Override + public void startedJob(DataVertical dataType, String exportService, String importService) { + TagContext tctx = tagger.emptyBuilder() + .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) + .put(KEY_EXPORT_SERVICE, TagValue.create(exportService), TAG_METADATA) + .put(KEY_IMPORT_SERVICE, TagValue.create(importService), TAG_METADATA) + .build(); + try (Scope ss = tagger.withTagContext(tctx)) { + STATS_RECORDER.newMeasureMap().put(JOB_STARTED, 1).record(); + } } - return INSTANCE; - } - - private GoogleDtpInternalMetricRecorder(GoogleCredentials credentials, String projectId) - throws IOException { - // Enable OpenCensus exporters to export metrics to Stackdriver Monitoring. - // Exporters use Application Default Credentials to authenticate. - // See https://developers.google.com/identity/protocols/application-default-credentials - // for more details. - StackdriverStatsConfiguration configuration = StackdriverStatsConfiguration.builder() - .setCredentials(credentials) - .setProjectId(projectId) - .setExportInterval(io.opencensus.common.Duration.create(EXPORT_INTERVAL_SECONDS, 0)) - .build(); - StackdriverStatsExporter.createAndRegister(configuration); - this.viewManager = Stats.getViewManager(); - setupViews(); - } - - private void setupViews() { - setupView(JOB_STARTED, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_IMPORT_SERVICE); - - setupView(JOB_FINISHED, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_IMPORT_SERVICE, KEY_SUCCESS); - setupView( - JOB_FINISHED_DURATION, - KEY_DATA_TYPE, - KEY_EXPORT_SERVICE, - KEY_IMPORT_SERVICE, - KEY_SUCCESS); - - setupView(EXPORT_PAGE_ATTEMPT, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); - setupView(EXPORT_PAGE_ATTEMPT_DURATION, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); - - setupView(EXPORT_PAGE, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); - setupView(EXPORT_PAGE_DURATION, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); - - setupView(IMPORT_PAGE_ATTEMPT, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); - setupView(IMPORT_PAGE_ATTEMPT_DURATION, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); - - setupView(IMPORT_PAGE, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); - setupView(IMPORT_PAGE_DURATION, KEY_DATA_TYPE, KEY_EXPORT_SERVICE, KEY_SUCCESS); - - setupView(GENERIC_COUNT, KEY_DATA_TYPE, KEY_GENERIC_SERVICE, KEY_GENERIC_TAG); - setupView( - GENERIC_BOOLEAN, - KEY_DATA_TYPE, - KEY_GENERIC_SERVICE, - KEY_GENERIC_TAG, - KEY_GENERIC_BOOL); - setupView(GENERIC_DURATION, KEY_DATA_TYPE, KEY_GENERIC_SERVICE, KEY_GENERIC_TAG); - } - - private void setupView(Measure measure, TagKey... keys) { - // Register the view. It is imperative that this step exists, - // otherwise recorded metrics will be dropped and never exported. - View view = View.create( - View.Name.create(NAME_BASE + measure.getName()), - measure.getDescription(), - measure, - Aggregation.Count.create(), - ImmutableList.copyOf(keys)); - - viewManager.registerView(view); - } - - @Override - public void startedJob(DataVertical dataType, String exportService, String importService) { - TagContext tctx = tagger.emptyBuilder() - .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) - .put(KEY_EXPORT_SERVICE, TagValue.create(exportService), TAG_METADATA) - .put(KEY_IMPORT_SERVICE, TagValue.create(importService), TAG_METADATA) - .build(); - try (Scope ss = tagger.withTagContext(tctx)) { - STATS_RECORDER.newMeasureMap().put(JOB_STARTED, 1).record(); + @Override + public void exportPageAttemptFinished( + DataVertical dataType, + ServiceResult serviceResult) { + TagContext tctx = tagger.emptyBuilder() + .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) + .put(KEY_EXPORT_SERVICE, TagValue.create(serviceResult.getService()), TAG_METADATA) + .put(KEY_SUCCESS, TagValue.create(Boolean.toString(serviceResult.getSuccess())), TAG_METADATA) + .build(); + try (Scope ss = tagger.withTagContext(tctx)) { + STATS_RECORDER.newMeasureMap() + .put(EXPORT_PAGE_ATTEMPT, 1) + .put(EXPORT_PAGE_ATTEMPT_DURATION, serviceResult.getDuration().toMillis()) + .record(); + } } - } - - @Override - public void exportPageAttemptFinished( - DataVertical dataType, - String service, - boolean success, - Duration duration) { - TagContext tctx = tagger.emptyBuilder() - .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) - .put(KEY_EXPORT_SERVICE, TagValue.create(service), TAG_METADATA) - .put(KEY_SUCCESS, TagValue.create(Boolean.toString(success)), TAG_METADATA) - .build(); - try (Scope ss = tagger.withTagContext(tctx)) { - STATS_RECORDER.newMeasureMap() - .put(EXPORT_PAGE_ATTEMPT, 1) - .put(EXPORT_PAGE_ATTEMPT_DURATION, duration.toMillis()) - .record(); + + @Override + public void exportPageFinished( + DataVertical dataType, + ServiceResult serviceResult) { + TagContext tctx = tagger.emptyBuilder() + .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) + .put(KEY_EXPORT_SERVICE, TagValue.create(serviceResult.getService()), TAG_METADATA) + .put(KEY_SUCCESS, TagValue.create(Boolean.toString(serviceResult.getSuccess())), TAG_METADATA) + .build(); + try (Scope ss = tagger.withTagContext(tctx)) { + STATS_RECORDER.newMeasureMap() + .put(EXPORT_PAGE, 1) + .put(EXPORT_PAGE_DURATION, serviceResult.getDuration().toMillis()) + .record(); + } } - } - - @Override - public void exportPageFinished( - DataVertical dataType, - String service, - boolean success, - Duration duration) { - TagContext tctx = tagger.emptyBuilder() - .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) - .put(KEY_EXPORT_SERVICE, TagValue.create(service), TAG_METADATA) - .put(KEY_SUCCESS, TagValue.create(Boolean.toString(success)), TAG_METADATA) - .build(); - try (Scope ss = tagger.withTagContext(tctx)) { - STATS_RECORDER.newMeasureMap() - .put(EXPORT_PAGE, 1) - .put(EXPORT_PAGE_DURATION, duration.toMillis()) - .record(); + + @Override + public void importPageAttemptFinished( + DataVertical dataType, + ServiceResult serviceResult) { + TagContext tctx = tagger.emptyBuilder() + .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) + .put(KEY_IMPORT_SERVICE, TagValue.create(serviceResult.getService()), TAG_METADATA) + .put(KEY_SUCCESS, TagValue.create(Boolean.toString(serviceResult.getSuccess())), TAG_METADATA) + .build(); + try (Scope ss = tagger.withTagContext(tctx)) { + STATS_RECORDER.newMeasureMap() + .put(IMPORT_PAGE_ATTEMPT, 1) + .put(IMPORT_PAGE_ATTEMPT_DURATION, serviceResult.getDuration().toMillis()) + .record(); + } } - } - - @Override - public void importPageAttemptFinished( - DataVertical dataType, - String service, - boolean success, - Duration duration) { - TagContext tctx = tagger.emptyBuilder() - .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) - .put(KEY_IMPORT_SERVICE, TagValue.create(service), TAG_METADATA) - .put(KEY_SUCCESS, TagValue.create(Boolean.toString(success)), TAG_METADATA) - .build(); - try (Scope ss = tagger.withTagContext(tctx)) { - STATS_RECORDER.newMeasureMap() - .put(IMPORT_PAGE_ATTEMPT, 1) - .put(IMPORT_PAGE_ATTEMPT_DURATION, duration.toMillis()) - .record(); + + @Override + public void importPageFinished( + DataVertical dataType, + ServiceResult serviceResult) { + TagContext tctx = tagger.emptyBuilder() + .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) + .put(KEY_IMPORT_SERVICE, TagValue.create(serviceResult.getService()), TAG_METADATA) + .put(KEY_SUCCESS, TagValue.create(Boolean.toString(serviceResult.getSuccess())), TAG_METADATA) + .build(); + try (Scope ss = tagger.withTagContext(tctx)) { + STATS_RECORDER.newMeasureMap() + .put(IMPORT_PAGE, 1) + .put(IMPORT_PAGE_DURATION, serviceResult.getDuration().toMillis()) + .record(); + } + } + + @Override + public void finishedJob( + DataVertical dataType, + String exportService, + String importService, + boolean success, + Duration duration) { + TagContext tctx = tagger.emptyBuilder() + .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) + .put(KEY_EXPORT_SERVICE, TagValue.create(exportService), TAG_METADATA) + .put(KEY_IMPORT_SERVICE, TagValue.create(importService), TAG_METADATA) + .put(KEY_SUCCESS, TagValue.create(Boolean.toString(success)), TAG_METADATA) + .build(); + try (Scope ss = tagger.withTagContext(tctx)) { + STATS_RECORDER.newMeasureMap() + .put(JOB_FINISHED, 1) + .put(JOB_FINISHED_DURATION, duration.toMillis()) + .record(); + } } - } - - @Override - public void importPageFinished( - DataVertical dataType, - String service, - boolean success, - Duration duration) { - TagContext tctx = tagger.emptyBuilder() - .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) - .put(KEY_IMPORT_SERVICE, TagValue.create(service), TAG_METADATA) - .put(KEY_SUCCESS, TagValue.create(Boolean.toString(success)), TAG_METADATA) - .build(); - try (Scope ss = tagger.withTagContext(tctx)) { - STATS_RECORDER.newMeasureMap() - .put(IMPORT_PAGE, 1) - .put(IMPORT_PAGE_DURATION, duration.toMillis()) - .record(); + + @Override + public void cancelledJob(DataVertical dataType, String exportService, String importService, Duration duration) { + // Need Google folks to implement the necessary changes here. } - } - - @Override - public void finishedJob( - DataVertical dataType, - String exportService, - String importService, - boolean success, - Duration duration) { - TagContext tctx = tagger.emptyBuilder() - .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) - .put(KEY_EXPORT_SERVICE, TagValue.create(exportService), TAG_METADATA) - .put(KEY_IMPORT_SERVICE, TagValue.create(importService), TAG_METADATA) - .put(KEY_SUCCESS, TagValue.create(Boolean.toString(success)), TAG_METADATA) - .build(); - try (Scope ss = tagger.withTagContext(tctx)) { - STATS_RECORDER.newMeasureMap() - .put(JOB_FINISHED, 1) - .put(JOB_FINISHED_DURATION, duration.toMillis()) - .record(); + + @Override + public void recordGenericMetric(DataVertical dataType, String service, String tag) { + recordGenericMetric(dataType, service, tag, 1); } - } - - @Override - public void cancelledJob(DataVertical dataType, String exportService, String importService, Duration duration) { - // Need Google folks to implement the necessary changes here. - } - - @Override - public void recordGenericMetric(DataVertical dataType, String service, String tag) { - recordGenericMetric(dataType, service, tag, 1); - } - - @Override - public void recordGenericMetric(DataVertical dataType, String service, String tag, boolean bool) { - TagContext tctx = tagger.emptyBuilder() - .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) - .put(KEY_GENERIC_SERVICE, TagValue.create(service), TAG_METADATA) - .put(KEY_GENERIC_TAG, TagValue.create(tag), TAG_METADATA) - .put(KEY_GENERIC_BOOL, TagValue.create(Boolean.toString(bool)), TAG_METADATA) - .build(); - try (Scope ss = tagger.withTagContext(tctx)) { - STATS_RECORDER.newMeasureMap() - .put(GENERIC_BOOLEAN, 1) - .record(); + + @Override + public void recordGenericMetric(DataVertical dataType, String service, String tag, boolean bool) { + TagContext tctx = tagger.emptyBuilder() + .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) + .put(KEY_GENERIC_SERVICE, TagValue.create(service), TAG_METADATA) + .put(KEY_GENERIC_TAG, TagValue.create(tag), TAG_METADATA) + .put(KEY_GENERIC_BOOL, TagValue.create(Boolean.toString(bool)), TAG_METADATA) + .build(); + try (Scope ss = tagger.withTagContext(tctx)) { + STATS_RECORDER.newMeasureMap() + .put(GENERIC_BOOLEAN, 1) + .record(); + } } - } - - @Override - public void recordGenericMetric(DataVertical dataType, String service, String tag, Duration duration) { - TagContext tctx = tagger.emptyBuilder() - .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) - .put(KEY_GENERIC_SERVICE, TagValue.create(service), TAG_METADATA) - .put(KEY_GENERIC_TAG, TagValue.create(tag), TAG_METADATA) - .build(); - try (Scope ss = tagger.withTagContext(tctx)) { - STATS_RECORDER.newMeasureMap() - .put(GENERIC_DURATION, duration.toMillis()) - .record(); + + @Override + public void recordGenericMetric(DataVertical dataType, String service, String tag, Duration duration) { + TagContext tctx = tagger.emptyBuilder() + .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) + .put(KEY_GENERIC_SERVICE, TagValue.create(service), TAG_METADATA) + .put(KEY_GENERIC_TAG, TagValue.create(tag), TAG_METADATA) + .build(); + try (Scope ss = tagger.withTagContext(tctx)) { + STATS_RECORDER.newMeasureMap() + .put(GENERIC_DURATION, duration.toMillis()) + .record(); + } } - } - - @Override - public void recordGenericMetric(DataVertical dataType, String service, String tag, int value) { - TagContext tctx = tagger.emptyBuilder() - .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) - .put(KEY_GENERIC_SERVICE, TagValue.create(service), TAG_METADATA) - .put(KEY_GENERIC_TAG, TagValue.create(tag), TAG_METADATA) - .build(); - try (Scope ss = tagger.withTagContext(tctx)) { - STATS_RECORDER.newMeasureMap() - .put(GENERIC_COUNT, value) - .record(); + + @Override + public void recordGenericMetric(DataVertical dataType, String service, String tag, int value) { + TagContext tctx = tagger.emptyBuilder() + .put(KEY_DATA_TYPE, TagValue.create(dataType.getDataType()), TAG_METADATA) + .put(KEY_GENERIC_SERVICE, TagValue.create(service), TAG_METADATA) + .put(KEY_GENERIC_TAG, TagValue.create(tag), TAG_METADATA) + .build(); + try (Scope ss = tagger.withTagContext(tctx)) { + STATS_RECORDER.newMeasureMap() + .put(GENERIC_COUNT, value) + .record(); + } } - } } diff --git a/extensions/cloud/portability-cloud-microsoft/src/main/java/org/datatransferproject/cloud/microsoft/cosmos/AzureDtpInternalMetricRecorder.java b/extensions/cloud/portability-cloud-microsoft/src/main/java/org/datatransferproject/cloud/microsoft/cosmos/AzureDtpInternalMetricRecorder.java index bdf714b9c..cbbd2e4da 100644 --- a/extensions/cloud/portability-cloud-microsoft/src/main/java/org/datatransferproject/cloud/microsoft/cosmos/AzureDtpInternalMetricRecorder.java +++ b/extensions/cloud/portability-cloud-microsoft/src/main/java/org/datatransferproject/cloud/microsoft/cosmos/AzureDtpInternalMetricRecorder.java @@ -15,143 +15,136 @@ */ package org.datatransferproject.cloud.microsoft.cosmos; -import static java.lang.String.format; - import org.datatransferproject.api.launcher.DtpInternalMetricRecorder; import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.api.launcher.ServiceResult; +import org.datatransferproject.types.common.models.DataVertical; import java.time.Duration; -import org.datatransferproject.types.common.models.DataVertical; + +import static java.lang.String.format; /** * A placeholder {@link DtpInternalMetricRecorder} that simply logs metrics * to the default monitor. - * **/ + **/ class AzureDtpInternalMetricRecorder implements DtpInternalMetricRecorder { - private final Monitor monitor; - - // TODO: Replace with a Azure Monitor based implementation. - AzureDtpInternalMetricRecorder(Monitor monitor) { - this.monitor = monitor; - } - - @Override - public void startedJob(DataVertical dataType, String exportService, String importService) { - monitor.debug( - () -> - format( - "Metric: StartedJob, data type: %s, from: %s, to: %s", - dataType, exportService, importService)); - } - - @Override - public void exportPageAttemptFinished( - DataVertical dataType, - String service, - boolean success, - Duration duration) { - monitor.debug( - () -> - format( - "Metric: exportPageAttemptFinished, data type: %s, service: %s, " - + "success: %s, duration: %s", - dataType, service, success, duration)); - } - - @Override - public void exportPageFinished( - DataVertical dataType, - String service, - boolean success, - Duration duration) { - monitor.debug( - () -> - format( - "Metric: exportPageFinished, data type: %s, service: %s, success: %s, duration: %s", - dataType, service, success, duration)); - } - - @Override - public void importPageAttemptFinished( - DataVertical dataType, - String service, - boolean success, - Duration duration) { - monitor.debug( - () -> - format( - "Metric: importPageAttemptFinished, data type: %s, service: %s," - + "success: %s, duration: %s", - dataType, service, success, duration)); - } - - @Override - public void importPageFinished( - DataVertical dataType, - String service, - boolean success, - Duration duration) { - monitor.debug( - () -> - format( - "Metric: importPageFinished, data type: %s, service: %s, success: %s, duration: %s", - dataType, service, success, duration)); - } - - @Override - public void finishedJob( - DataVertical dataType, - String exportService, - String importService, - boolean success, - Duration duration) { - monitor.debug( - () -> - format( - "Metric: finishedJob, data type: %s, from: %s, to: %s, success: %s, duration: %s", - dataType, exportService, importService, success, duration)); - } - - @Override - public void cancelledJob(DataVertical dataType, String exportService, String importService, Duration duration) { - monitor.debug( - () -> - format("Metric: cancelledJob, data type: %s, from: %s, to: %s, duration: %s", - dataType, exportService, importService, duration)); - - } - - @Override - public void recordGenericMetric(DataVertical dataType, String service, String tag) { - monitor.debug( - () -> - format("Metric: Generic, data type: %s, service: %s, tag: %s", dataType, service, tag)); - } - - @Override - public void recordGenericMetric(DataVertical dataType, String service, String tag, boolean bool) { - monitor.debug( - () -> - format( - "Metric: Generic, data type: %s, service: %s, tag: %s, value: %s", - dataType, service, tag, bool)); - } - - @Override - public void recordGenericMetric(DataVertical dataType, String service, String tag, Duration duration) { - monitor.debug( - () -> - format( - "Metric: Generic, data type: %s, service: %s, tag: %s, duration: %s", - dataType, service, tag, duration)); - } - - @Override - public void recordGenericMetric(DataVertical dataType, String service, String tag, int value) { - monitor.debug( - () -> - format( - "Metric: Generic, data type: %s, service: %s, tag: %s, value: %s", - dataType, service, tag, value)); - } + private final Monitor monitor; + + // TODO: Replace with a Azure Monitor based implementation. + AzureDtpInternalMetricRecorder(Monitor monitor) { + this.monitor = monitor; + } + + @Override + public void startedJob(DataVertical dataType, String exportService, String importService) { + monitor.debug( + () -> + format( + "Metric: StartedJob, data type: %s, from: %s, to: %s", + dataType, exportService, importService)); + } + + @Override + public void exportPageAttemptFinished( + DataVertical dataType, + ServiceResult serviceResult) { + monitor.debug( + () -> + format( + "Metric: exportPageAttemptFinished, data type: %s, service: %s, " + + "success: %s, duration: %s", + dataType, serviceResult.getService(), serviceResult.getSuccess(), serviceResult.getDuration())); + } + + @Override + public void exportPageFinished( + DataVertical dataType, + ServiceResult serviceResult) { + monitor.debug( + () -> + format( + "Metric: exportPageFinished, data type: %s, service: %s, success: %s, duration: %s", + dataType, serviceResult.getService(), serviceResult.getSuccess(), serviceResult.getDuration())); + } + + @Override + public void importPageAttemptFinished( + DataVertical dataType, + ServiceResult serviceResult) { + monitor.debug( + () -> + format( + "Metric: importPageAttemptFinished, data type: %s, service: %s," + + "success: %s, duration: %s", + dataType, serviceResult.getService(), serviceResult.getSuccess(), serviceResult.getDuration())); + } + + @Override + public void importPageFinished( + DataVertical dataType, + ServiceResult serviceResult) { + monitor.debug( + () -> + format( + "Metric: importPageFinished, data type: %s, service: %s, success: %s, duration: %s", + dataType, serviceResult.getService(), serviceResult.getSuccess(), serviceResult.getDuration())); + } + + @Override + public void finishedJob( + DataVertical dataType, + String exportService, + String importService, + boolean success, + Duration duration) { + monitor.debug( + () -> + format( + "Metric: finishedJob, data type: %s, from: %s, to: %s, success: %s, duration: %s", + dataType, exportService, importService, success, duration)); + } + + @Override + public void cancelledJob(DataVertical dataType, String exportService, String importService, Duration duration) { + monitor.debug( + () -> + format("Metric: cancelledJob, data type: %s, from: %s, to: %s, duration: %s", + dataType, exportService, importService, duration)); + + } + + @Override + public void recordGenericMetric(DataVertical dataType, String service, String tag) { + monitor.debug( + () -> + format("Metric: Generic, data type: %s, service: %s, tag: %s", dataType, service, tag)); + } + + @Override + public void recordGenericMetric(DataVertical dataType, String service, String tag, boolean bool) { + monitor.debug( + () -> + format( + "Metric: Generic, data type: %s, service: %s, tag: %s, value: %s", + dataType, service, tag, bool)); + } + + @Override + public void recordGenericMetric(DataVertical dataType, String service, String tag, Duration duration) { + monitor.debug( + () -> + format( + "Metric: Generic, data type: %s, service: %s, tag: %s, duration: %s", + dataType, service, tag, duration)); + } + + @Override + public void recordGenericMetric(DataVertical dataType, String service, String tag, int value) { + monitor.debug( + () -> + format( + "Metric: Generic, data type: %s, service: %s, tag: %s, value: %s", + dataType, service, tag, value)); + } } diff --git a/portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/DtpInternalMetricRecorder.java b/portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/DtpInternalMetricRecorder.java index 2c39c292f..62879194b 100644 --- a/portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/DtpInternalMetricRecorder.java +++ b/portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/DtpInternalMetricRecorder.java @@ -15,9 +15,10 @@ */ package org.datatransferproject.api.launcher; -import java.time.Duration; import org.datatransferproject.types.common.models.DataVertical; +import java.time.Duration; + /** * Interface to log metrics about a DTP job. * @@ -28,47 +29,70 @@ * platform. */ public interface DtpInternalMetricRecorder { - // Metrics related to DTP internals + // Metrics related to DTP internals + + /** + * A DTP job started. + **/ + void startedJob(DataVertical dataType, String exportService, String importService); + + /** + * A DTP job finished + **/ + void finishedJob( + DataVertical dataType, + String exportService, + String importService, + boolean success, + Duration duration); + + /** + * A DTP job cancelled + **/ + void cancelledJob( + DataVertical dataType, + String exportService, + String importService, + Duration duration); + + /** + * An single attempt to export a page of data finished. + * + * @param serviceResult The ServiceResult object + */ + void exportPageAttemptFinished( + DataVertical dataType, + ServiceResult serviceResult); + + /** + * An attempt to export a page of data finished including all retires. + * + * @param serviceResult The ServiceResult object + */ + void exportPageFinished(DataVertical dataType, ServiceResult serviceResult); - /** A DTP job started. **/ - void startedJob(DataVertical dataType, String exportService, String importService); - /** A DTP job finished **/ - void finishedJob( - DataVertical dataType, - String exportService, - String importService, - boolean success, - Duration duration); - /** A DTP job cancelled **/ - void cancelledJob( - DataVertical dataType, - String exportService, - String importService, - Duration duration); + /** + * An single attempt to import a page of data finished. + * + * @param serviceResult The ServiceResult object + */ + void importPageAttemptFinished( + DataVertical dataType, + ServiceResult serviceResult); - /** An single attempt to export a page of data finished. **/ - void exportPageAttemptFinished( - DataVertical dataType, - String service, - boolean success, - Duration duration); + /** + * An attempt to import a page of data finished including all retires. + * + * @param serviceResult The ServiceResult object + **/ + void importPageFinished(DataVertical dataType, ServiceResult serviceResult); - /** An attempt to export a page of data finished including all retires. **/ - void exportPageFinished(DataVertical dataType, String service, boolean success, Duration duration); + // Metrics from {@link MetricRecorder} + void recordGenericMetric(DataVertical dataType, String service, String tag); - /** An single attempt to import a page of data finished. **/ - void importPageAttemptFinished( - DataVertical dataType, - String service, - boolean success, - Duration duration); + void recordGenericMetric(DataVertical dataType, String service, String tag, boolean bool); - /** An attempt to import a page of data finished including all retires. **/ - void importPageFinished(DataVertical dataType, String service, boolean success, Duration duration); + void recordGenericMetric(DataVertical dataType, String service, String tag, Duration duration); - // Metrics from {@link MetricRecorder} - void recordGenericMetric(DataVertical dataType, String service, String tag); - void recordGenericMetric(DataVertical dataType, String service, String tag, boolean bool); - void recordGenericMetric(DataVertical dataType, String service, String tag, Duration duration); - void recordGenericMetric(DataVertical dataType, String service, String tag, int value); + void recordGenericMetric(DataVertical dataType, String service, String tag, int value); } diff --git a/portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/ServiceResult.java b/portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/ServiceResult.java new file mode 100644 index 000000000..e50bc8df8 --- /dev/null +++ b/portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/ServiceResult.java @@ -0,0 +1,39 @@ +package org.datatransferproject.api.launcher; +public class ServiceResult{ + private java.lang.String service; + + public java.lang.String getService(){ + return service; + } + + public void setService(java.lang.String service){ + this.service=service; + } + + private boolean success; + + public boolean getSuccess(){ + return success; + } + + public void setSuccess(boolean success){ + this.success=success; + } + + private java.time.Duration duration; + + public java.time.Duration getDuration(){ + return duration; + } + + public void setDuration(java.time.Duration duration){ + this.duration=duration; + } + + public ServiceResult(java.lang.String service,boolean success,java.time.Duration duration){ + this.service=service; + this.success=success; + this.duration=duration; + } +} + diff --git a/portability-api-launcher/src/main/java/org/datatransferproject/launcher/metrics/LoggingDtpInternalMetricRecorder.java b/portability-api-launcher/src/main/java/org/datatransferproject/launcher/metrics/LoggingDtpInternalMetricRecorder.java index 1d40540c1..f753a8b45 100644 --- a/portability-api-launcher/src/main/java/org/datatransferproject/launcher/metrics/LoggingDtpInternalMetricRecorder.java +++ b/portability-api-launcher/src/main/java/org/datatransferproject/launcher/metrics/LoggingDtpInternalMetricRecorder.java @@ -15,156 +15,149 @@ */ package org.datatransferproject.launcher.metrics; -import static java.lang.String.format; - import org.datatransferproject.api.launcher.DtpInternalMetricRecorder; import org.datatransferproject.api.launcher.ExtensionContext; import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.api.launcher.ServiceResult; +import org.datatransferproject.types.common.models.DataVertical; import java.time.Duration; -import org.datatransferproject.types.common.models.DataVertical; + +import static java.lang.String.format; /** * A default {@link DtpInternalMetricRecorder} that simply logs metrics * to the default monitor. - * **/ + **/ public class LoggingDtpInternalMetricRecorder implements DtpInternalMetricRecorder { - private final Monitor monitor; - - /** - * Registers a LoggingDtpInternalMetricRecorder in the {@link ExtensionContext} if there is not - * another {@link DtpInternalMetricRecorder} registered. - **/ - public static void registerRecorderIfNeeded(ExtensionContext context) { - if (context.getService(DtpInternalMetricRecorder.class) == null) { - context.registerService( - DtpInternalMetricRecorder.class, - new LoggingDtpInternalMetricRecorder(context.getMonitor())); + private final Monitor monitor; + + /** + * Registers a LoggingDtpInternalMetricRecorder in the {@link ExtensionContext} if there is not + * another {@link DtpInternalMetricRecorder} registered. + **/ + public static void registerRecorderIfNeeded(ExtensionContext context) { + if (context.getService(DtpInternalMetricRecorder.class) == null) { + context.registerService( + DtpInternalMetricRecorder.class, + new LoggingDtpInternalMetricRecorder(context.getMonitor())); + } + } + + private LoggingDtpInternalMetricRecorder(Monitor monitor) { + this.monitor = monitor; + } + + @Override + public void startedJob(DataVertical dataType, String exportService, String importService) { + monitor.debug( + () -> + format( + "Metric: StartedJob, data type: %s, from: %s, to: %s", + dataType, exportService, importService)); + } + + @Override + public void exportPageAttemptFinished( + DataVertical dataType, + ServiceResult serviceResult) { + monitor.debug( + () -> + format( + "Metric: exportPageAttemptFinished, data type: %s, service: %s, " + + "success: %s, duration: %s", + dataType, serviceResult.getService(), serviceResult.getSuccess(), serviceResult.getDuration())); + } + + @Override + public void exportPageFinished( + DataVertical dataType, + ServiceResult serviceResult) { + monitor.debug( + () -> + format( + "Metric: exportPageFinished, data type: %s, service: %s, success: %s, duration: %s", + dataType, serviceResult.getService(), serviceResult.getSuccess(), serviceResult.getDuration())); + } + + @Override + public void importPageAttemptFinished( + DataVertical dataType, + ServiceResult serviceResult) { + monitor.debug( + () -> + format( + "Metric: importPageAttemptFinished, data type: %s, service: %s," + + "success: %s, duration: %s", + dataType, serviceResult.getService(), serviceResult.getSuccess(), serviceResult.getDuration())); + } + + @Override + public void importPageFinished( + DataVertical dataType, + ServiceResult serviceResult) { + monitor.debug( + () -> + format( + "Metric: importPageFinished, data type: %s, service: %s, success: %s, duration: %s", + dataType, serviceResult.getService(), serviceResult.getSuccess(), serviceResult.getDuration())); + } + + @Override + public void finishedJob( + DataVertical dataType, + String exportService, + String importService, + boolean success, + Duration duration) { + monitor.debug( + () -> + format( + "Metric: finishedJob, data type: %s, from: %s, to: %s, success: %s, duration: %s", + dataType, exportService, importService, success, duration)); + } + + @Override + public void cancelledJob( + DataVertical dataType, String exportService, String importService, Duration duration) { + monitor.debug( + () -> + format( + "Metric: cancelledJob, data type: %s, from: %s, to: %s, duration: %s", + dataType, exportService, importService, duration)); + } + + @Override + public void recordGenericMetric(DataVertical dataType, String service, String tag) { + monitor.debug( + () -> + format("Metric: Generic, data type: %s, service: %s, tag: %s", dataType, service, tag)); + } + + @Override + public void recordGenericMetric(DataVertical dataType, String service, String tag, boolean bool) { + monitor.debug( + () -> + format( + "Metric: Generic, data type: %s, service: %s, tag: %s, value: %s", + dataType, service, tag, bool)); + } + + @Override + public void recordGenericMetric(DataVertical dataType, String service, String tag, Duration duration) { + monitor.debug( + () -> + format( + "Metric: Generic, data type: %s, service: %s, tag: %s, duration: %s", + dataType, service, tag, duration)); + } + + @Override + public void recordGenericMetric(DataVertical dataType, String service, String tag, int value) { + monitor.debug( + () -> + format( + "Metric: Generic, data type: %s, service: %s, tag: %s, value: %s", + dataType, service, tag, value)); } - } - - private LoggingDtpInternalMetricRecorder(Monitor monitor) { - this.monitor = monitor; - } - - @Override - public void startedJob(DataVertical dataType, String exportService, String importService) { - monitor.debug( - () -> - format( - "Metric: StartedJob, data type: %s, from: %s, to: %s", - dataType, exportService, importService)); - } - - @Override - public void exportPageAttemptFinished( - DataVertical dataType, - String service, - boolean success, - Duration duration) { - monitor.debug( - () -> - format( - "Metric: exportPageAttemptFinished, data type: %s, service: %s, " - + "success: %s, duration: %s", - dataType, service, success, duration)); - } - - @Override - public void exportPageFinished( - DataVertical dataType, - String service, - boolean success, - Duration duration) { - monitor.debug( - () -> - format( - "Metric: exportPageFinished, data type: %s, service: %s, success: %s, duration: %s", - dataType, service, success, duration)); - } - - @Override - public void importPageAttemptFinished( - DataVertical dataType, - String service, - boolean success, - Duration duration) { - monitor.debug( - () -> - format( - "Metric: importPageAttemptFinished, data type: %s, service: %s," - + "success: %s, duration: %s", - dataType, service, success, duration)); - } - - @Override - public void importPageFinished( - DataVertical dataType, - String service, - boolean success, - Duration duration) { - monitor.debug( - () -> - format( - "Metric: importPageFinished, data type: %s, service: %s, success: %s, duration: %s", - dataType, service, success, duration)); - } - - @Override - public void finishedJob( - DataVertical dataType, - String exportService, - String importService, - boolean success, - Duration duration) { - monitor.debug( - () -> - format( - "Metric: finishedJob, data type: %s, from: %s, to: %s, success: %s, duration: %s", - dataType, exportService, importService, success, duration)); - } - - @Override - public void cancelledJob( - DataVertical dataType, String exportService, String importService, Duration duration) { - monitor.debug( - () -> - format( - "Metric: cancelledJob, data type: %s, from: %s, to: %s, duration: %s", - dataType, exportService, importService, duration)); - } - - @Override - public void recordGenericMetric(DataVertical dataType, String service, String tag) { - monitor.debug( - () -> - format("Metric: Generic, data type: %s, service: %s, tag: %s", dataType, service, tag)); - } - - @Override - public void recordGenericMetric(DataVertical dataType, String service, String tag, boolean bool) { - monitor.debug( - () -> - format( - "Metric: Generic, data type: %s, service: %s, tag: %s, value: %s", - dataType, service, tag, bool)); - } - - @Override - public void recordGenericMetric(DataVertical dataType, String service, String tag, Duration duration) { - monitor.debug( - () -> - format( - "Metric: Generic, data type: %s, service: %s, tag: %s, duration: %s", - dataType, service, tag, duration)); - } - - @Override - public void recordGenericMetric(DataVertical dataType, String service, String tag, int value) { - monitor.debug( - () -> - format( - "Metric: Generic, data type: %s, service: %s, tag: %s, value: %s", - dataType, service, tag, value)); - } } diff --git a/portability-transfer/src/main/java/org/datatransferproject/transfer/CallableExporter.java b/portability-transfer/src/main/java/org/datatransferproject/transfer/CallableExporter.java index 9e6a4b8f8..6b3a5dd84 100644 --- a/portability-transfer/src/main/java/org/datatransferproject/transfer/CallableExporter.java +++ b/portability-transfer/src/main/java/org/datatransferproject/transfer/CallableExporter.java @@ -19,6 +19,7 @@ import com.google.common.base.Stopwatch; import com.google.inject.Provider; import org.datatransferproject.api.launcher.DtpInternalMetricRecorder; +import org.datatransferproject.api.launcher.ServiceResult; import org.datatransferproject.spi.transfer.provider.ExportResult; import org.datatransferproject.spi.transfer.provider.Exporter; import org.datatransferproject.types.common.ExportInformation; @@ -35,39 +36,40 @@ */ public class CallableExporter implements Callable { - private Provider exporterProvider; - private UUID jobId; - private AuthData authData; - private Optional exportInformation; - private final DtpInternalMetricRecorder metricRecorder; + private Provider exporterProvider; + private UUID jobId; + private AuthData authData; + private Optional exportInformation; + private final DtpInternalMetricRecorder metricRecorder; - public CallableExporter( - Provider exporterProvider, - UUID jobId, - AuthData authData, - Optional exportInformation, - DtpInternalMetricRecorder metricRecorder) { - this.exporterProvider = checkNotNull(exporterProvider, "exportProvider can't be null"); - this.jobId = checkNotNull(jobId, "jobId can't be null"); - this.authData = checkNotNull(authData, "authData can't be null"); - this.exportInformation = exportInformation; - this.metricRecorder = checkNotNull(metricRecorder, "metric recorder can't be null"); - } + public CallableExporter( + Provider exporterProvider, + UUID jobId, + AuthData authData, + Optional exportInformation, + DtpInternalMetricRecorder metricRecorder) { + this.exporterProvider = checkNotNull(exporterProvider, "exportProvider can't be null"); + this.jobId = checkNotNull(jobId, "jobId can't be null"); + this.authData = checkNotNull(authData, "authData can't be null"); + this.exportInformation = exportInformation; + this.metricRecorder = checkNotNull(metricRecorder, "metric recorder can't be null"); + } - @Override - public ExportResult call() throws Exception { - boolean success = false; - Stopwatch stopwatch = Stopwatch.createStarted(); - try { - ExportResult result = exporterProvider.get().export(jobId, authData, exportInformation); - success = result.getType() != ExportResult.ResultType.ERROR; - return result; - } finally{ - metricRecorder.exportPageAttemptFinished( - JobMetadata.getDataType(), - JobMetadata.getExportService(), - success, - stopwatch.elapsed()); + @Override + public ExportResult call() throws Exception { + boolean success = false; + Stopwatch stopwatch = Stopwatch.createStarted(); + try { + ExportResult result = exporterProvider.get().export(jobId, authData, exportInformation); + success = result.getType() != ExportResult.ResultType.ERROR; + return result; + } finally { + metricRecorder.exportPageAttemptFinished( + JobMetadata.getDataType(), new ServiceResult( + JobMetadata.getExportService(), + success, + stopwatch.elapsed()) + ); + } } - } } diff --git a/portability-transfer/src/main/java/org/datatransferproject/transfer/CallableImporter.java b/portability-transfer/src/main/java/org/datatransferproject/transfer/CallableImporter.java index 6610cc5f3..cd5811eda 100644 --- a/portability-transfer/src/main/java/org/datatransferproject/transfer/CallableImporter.java +++ b/portability-transfer/src/main/java/org/datatransferproject/transfer/CallableImporter.java @@ -19,6 +19,7 @@ import com.google.common.base.Stopwatch; import com.google.inject.Provider; import org.datatransferproject.api.launcher.DtpInternalMetricRecorder; +import org.datatransferproject.api.launcher.ServiceResult; import org.datatransferproject.spi.transfer.idempotentexecutor.IdempotentImportExecutor; import org.datatransferproject.spi.transfer.provider.ImportResult; import org.datatransferproject.spi.transfer.provider.Importer; @@ -36,55 +37,56 @@ */ public class CallableImporter implements Callable { - private final Provider importerProvider; - private final UUID jobId; - private final IdempotentImportExecutor idempotentImportExecutor; - private final AuthData authData; - private final DataModel data; - private final DtpInternalMetricRecorder metricRecorder; + private final Provider importerProvider; + private final UUID jobId; + private final IdempotentImportExecutor idempotentImportExecutor; + private final AuthData authData; + private final DataModel data; + private final DtpInternalMetricRecorder metricRecorder; - public CallableImporter( - Provider importerProvider, - UUID jobId, - IdempotentImportExecutor idempotentImportExecutor, - AuthData authData, - DataModel data, - DtpInternalMetricRecorder metricRecorder) { - this.importerProvider = importerProvider; - this.jobId = jobId; - this.idempotentImportExecutor = idempotentImportExecutor; - this.authData = authData; - this.data = data; - this.metricRecorder = metricRecorder; - } + public CallableImporter( + Provider importerProvider, + UUID jobId, + IdempotentImportExecutor idempotentImportExecutor, + AuthData authData, + DataModel data, + DtpInternalMetricRecorder metricRecorder) { + this.importerProvider = importerProvider; + this.jobId = jobId; + this.idempotentImportExecutor = idempotentImportExecutor; + this.authData = authData; + this.data = data; + this.metricRecorder = metricRecorder; + } - @Override - public ImportResult call() throws Exception { - boolean success = false; - Stopwatch stopwatch = Stopwatch.createStarted(); - try { - idempotentImportExecutor.resetRecentErrors(); - ImportResult result = importerProvider.get() - .importItem(jobId, idempotentImportExecutor, authData, data); + @Override + public ImportResult call() throws Exception { + boolean success = false; + Stopwatch stopwatch = Stopwatch.createStarted(); + try { + idempotentImportExecutor.resetRecentErrors(); + ImportResult result = importerProvider.get() + .importItem(jobId, idempotentImportExecutor, authData, data); - Collection errors = idempotentImportExecutor.getRecentErrors(); - success = result.getType() == ImportResult.ResultType.OK && errors.isEmpty(); + Collection errors = idempotentImportExecutor.getRecentErrors(); + success = result.getType() == ImportResult.ResultType.OK && errors.isEmpty(); - if (!success && errors.iterator().hasNext() && !errors.iterator().next().canSkip()) { - throw new IOException( - "Problem with importer, forcing a retry, " - + "first error: " - + (errors.iterator().hasNext() ? errors.iterator().next().exception() : "none")); - } + if (!success && errors.iterator().hasNext() && !errors.iterator().next().canSkip()) { + throw new IOException( + "Problem with importer, forcing a retry, " + + "first error: " + + (errors.iterator().hasNext() ? errors.iterator().next().exception() : "none")); + } - result = result.copyWithCounts(data.getCounts()); - return result; - } finally{ - metricRecorder.importPageAttemptFinished( - JobMetadata.getDataType(), - JobMetadata.getImportService(), - success, - stopwatch.elapsed()); + result = result.copyWithCounts(data.getCounts()); + return result; + } finally { + metricRecorder.importPageAttemptFinished( + JobMetadata.getDataType(), new ServiceResult( + JobMetadata.getImportService(), + success, + stopwatch.elapsed()) + ); + } } - } } diff --git a/portability-transfer/src/main/java/org/datatransferproject/transfer/copier/PortabilityAbstractInMemoryDataCopier.java b/portability-transfer/src/main/java/org/datatransferproject/transfer/copier/PortabilityAbstractInMemoryDataCopier.java index e0c88d0f9..87e2249cb 100644 --- a/portability-transfer/src/main/java/org/datatransferproject/transfer/copier/PortabilityAbstractInMemoryDataCopier.java +++ b/portability-transfer/src/main/java/org/datatransferproject/transfer/copier/PortabilityAbstractInMemoryDataCopier.java @@ -18,16 +18,9 @@ import com.google.cloud.datastore.DatastoreException; import com.google.common.base.Stopwatch; import com.google.inject.Provider; -import java.io.IOException; -import java.time.Clock; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Optional; -import java.util.UUID; import org.datatransferproject.api.launcher.DtpInternalMetricRecorder; import org.datatransferproject.api.launcher.Monitor; +import org.datatransferproject.api.launcher.ServiceResult; import org.datatransferproject.launcher.monitor.events.EventCode; import org.datatransferproject.spi.cloud.connection.ConnectionProvider; import org.datatransferproject.spi.cloud.storage.JobStore; @@ -40,11 +33,7 @@ import org.datatransferproject.spi.transfer.provider.Importer; import org.datatransferproject.spi.transfer.types.CopyException; import org.datatransferproject.spi.transfer.types.CopyExceptionWithFailureReason; -import org.datatransferproject.transfer.Annotations; -import org.datatransferproject.transfer.CallableExporter; -import org.datatransferproject.transfer.CallableImporter; -import org.datatransferproject.transfer.CallableSizeCalculator; -import org.datatransferproject.transfer.JobMetadata; +import org.datatransferproject.transfer.*; import org.datatransferproject.types.common.DownloadableItem; import org.datatransferproject.types.common.ExportInformation; import org.datatransferproject.types.common.models.DataModel; @@ -57,233 +46,241 @@ import org.datatransferproject.types.transfer.retry.RetryStrategyLibrary; import org.datatransferproject.types.transfer.retry.RetryingCallable; +import java.io.IOException; +import java.time.Clock; +import java.util.*; + public abstract class PortabilityAbstractInMemoryDataCopier implements InMemoryDataCopier { - /** - * Lazy evaluate exporter and importer as their providers depend on the polled {@code - * PortabilityJob} which is not available at startup. - */ - protected final Provider exporterProvider; + /** + * Lazy evaluate exporter and importer as their providers depend on the polled {@code + * PortabilityJob} which is not available at startup. + */ + protected final Provider exporterProvider; - protected final Provider importerProvider; - protected final IdempotentImportExecutor idempotentImportExecutor; - protected final IdempotentImportExecutor retryingIdempotentImportExecutor; - protected final Provider retryStrategyLibraryProvider; - protected final Monitor monitor; - protected final DtpInternalMetricRecorder metricRecorder; - protected final JobStore jobStore; + protected final Provider importerProvider; + protected final IdempotentImportExecutor idempotentImportExecutor; + protected final IdempotentImportExecutor retryingIdempotentImportExecutor; + protected final Provider retryStrategyLibraryProvider; + protected final Monitor monitor; + protected final DtpInternalMetricRecorder metricRecorder; + protected final JobStore jobStore; - public PortabilityAbstractInMemoryDataCopier( - Provider exporterProvider, - Provider importerProvider, - Provider retryStrategyLibraryProvider, - Monitor monitor, - IdempotentImportExecutor idempotentImportExecutor, - @Annotations.RetryingExecutor IdempotentImportExecutor retryingIdempotentImportExecutor, - DtpInternalMetricRecorder dtpInternalMetricRecorder, - JobStore jobStore) { - this.exporterProvider = exporterProvider; - this.importerProvider = importerProvider; - this.retryStrategyLibraryProvider = retryStrategyLibraryProvider; - this.monitor = monitor; - this.idempotentImportExecutor = idempotentImportExecutor; - this.retryingIdempotentImportExecutor = retryingIdempotentImportExecutor; - this.metricRecorder = dtpInternalMetricRecorder; - this.jobStore = jobStore; - } + public PortabilityAbstractInMemoryDataCopier( + Provider exporterProvider, + Provider importerProvider, + Provider retryStrategyLibraryProvider, + Monitor monitor, + IdempotentImportExecutor idempotentImportExecutor, + @Annotations.RetryingExecutor IdempotentImportExecutor retryingIdempotentImportExecutor, + DtpInternalMetricRecorder dtpInternalMetricRecorder, + JobStore jobStore) { + this.exporterProvider = exporterProvider; + this.importerProvider = importerProvider; + this.retryStrategyLibraryProvider = retryStrategyLibraryProvider; + this.monitor = monitor; + this.idempotentImportExecutor = idempotentImportExecutor; + this.retryingIdempotentImportExecutor = retryingIdempotentImportExecutor; + this.metricRecorder = dtpInternalMetricRecorder; + this.jobStore = jobStore; + } - public abstract void resetCopyIterationCounter(); + public abstract void resetCopyIterationCounter(); - /** Kicks off transfer job {@code jobId} from {@code exporter} to {@code importer}. */ - @Override - public abstract void copy( - AuthData exportAuthData, - AuthData importAuthData, - UUID jobId, - Optional exportInfo) - throws IOException, CopyException; + /** + * Kicks off transfer job {@code jobId} from {@code exporter} to {@code importer}. + */ + @Override + public abstract void copy( + AuthData exportAuthData, + AuthData importAuthData, + UUID jobId, + Optional exportInfo) + throws IOException, CopyException; - @Override - public Collection getErrors(UUID jobId) { - idempotentImportExecutor.setJobId(jobId); - retryingIdempotentImportExecutor.setJobId(jobId); - return idempotentImportExecutor.getErrors(); - } + @Override + public Collection getErrors(UUID jobId) { + idempotentImportExecutor.setJobId(jobId); + retryingIdempotentImportExecutor.setJobId(jobId); + return idempotentImportExecutor.getErrors(); + } - protected ExportResult copyIteration( - UUID jobId, - AuthData exportAuthData, - AuthData importAuthData, - Optional exportInformation, - String jobIdPrefix, - int copyIteration) - throws CopyException { - monitor.debug(() -> jobIdPrefix + "Copy iteration: " + copyIteration); + protected ExportResult copyIteration( + UUID jobId, + AuthData exportAuthData, + AuthData importAuthData, + Optional exportInformation, + String jobIdPrefix, + int copyIteration) + throws CopyException { + monitor.debug(() -> jobIdPrefix + "Copy iteration: " + copyIteration); - ExportResult exportResult = - exportIteration(jobId, exportAuthData, exportInformation, jobIdPrefix, copyIteration); + ExportResult exportResult = + exportIteration(jobId, exportAuthData, exportInformation, jobIdPrefix, copyIteration); - DataModel exportedData = exportResult.getExportedData(); - if (exportedData != null) { - PortabilityJob job = jobStore.findJob(jobId); - TransferMode transferMode = - job.transferMode() == null ? TransferMode.DATA_TRANSFER : job.transferMode(); - switch (transferMode) { - case DATA_TRANSFER: - importIteration(jobId, importAuthData, jobIdPrefix, copyIteration, exportedData); - break; - case SIZE_CALCULATION: - sizeCalculationIteration(jobId, jobIdPrefix, exportedData); - break; - default: - throw new IllegalStateException( - "Job mode " + transferMode.name() + " is not supported by " - + getClass().getSimpleName()); - } - } + DataModel exportedData = exportResult.getExportedData(); + if (exportedData != null) { + PortabilityJob job = jobStore.findJob(jobId); + TransferMode transferMode = + job.transferMode() == null ? TransferMode.DATA_TRANSFER : job.transferMode(); + switch (transferMode) { + case DATA_TRANSFER: + importIteration(jobId, importAuthData, jobIdPrefix, copyIteration, exportedData); + break; + case SIZE_CALCULATION: + sizeCalculationIteration(jobId, jobIdPrefix, exportedData); + break; + default: + throw new IllegalStateException( + "Job mode " + transferMode.name() + " is not supported by " + + getClass().getSimpleName()); + } + } - return exportResult; - } + return exportResult; + } - private ExportResult exportIteration( - UUID jobId, - AuthData exportAuthData, - Optional exportInformation, - String jobIdPrefix, - int copyIteration) - throws CopyException { + private ExportResult exportIteration( + UUID jobId, + AuthData exportAuthData, + Optional exportInformation, + String jobIdPrefix, + int copyIteration) + throws CopyException { - monitor.debug( - () -> jobIdPrefix + "Starting export, copy iteration: " + copyIteration, - EventCode.COPIER_STARTED_EXPORT); + monitor.debug( + () -> jobIdPrefix + "Starting export, copy iteration: " + copyIteration, + EventCode.COPIER_STARTED_EXPORT); - CallableExporter callableExporter = - new CallableExporter( - exporterProvider, jobId, exportAuthData, exportInformation, metricRecorder); - RetryingCallable retryingExporter = - new RetryingCallable<>( - callableExporter, - retryStrategyLibraryProvider.get(), - Clock.systemUTC(), - monitor, - JobMetadata.getDataType(), - JobMetadata.getExportService()); - boolean exportSuccess = false; - Stopwatch exportStopwatch = Stopwatch.createStarted(); - try { - ExportResult exportResult = retryingExporter.call(); - exportSuccess = exportResult.getType() != ExportResult.ResultType.ERROR; - monitor.debug( - () -> jobIdPrefix + "Finished export, copy iteration: " + copyIteration, - EventCode.COPIER_FINISHED_EXPORT); - return exportResult; - } catch (RetryException | RuntimeException e) { - throw convertToCopyException(jobIdPrefix, "export", e); - } finally { - metricRecorder.exportPageFinished( - JobMetadata.getDataType(), - JobMetadata.getExportService(), - exportSuccess, - exportStopwatch.elapsed()); + CallableExporter callableExporter = + new CallableExporter( + exporterProvider, jobId, exportAuthData, exportInformation, metricRecorder); + RetryingCallable retryingExporter = + new RetryingCallable<>( + callableExporter, + retryStrategyLibraryProvider.get(), + Clock.systemUTC(), + monitor, + JobMetadata.getDataType(), + JobMetadata.getExportService()); + boolean exportSuccess = false; + Stopwatch exportStopwatch = Stopwatch.createStarted(); + try { + ExportResult exportResult = retryingExporter.call(); + exportSuccess = exportResult.getType() != ExportResult.ResultType.ERROR; + monitor.debug( + () -> jobIdPrefix + "Finished export, copy iteration: " + copyIteration, + EventCode.COPIER_FINISHED_EXPORT); + return exportResult; + } catch (RetryException | RuntimeException e) { + throw convertToCopyException(jobIdPrefix, "export", e); + } finally { + metricRecorder.exportPageFinished( + JobMetadata.getDataType(), new ServiceResult( + JobMetadata.getExportService(), + exportSuccess, + exportStopwatch.elapsed()) + ); + } } - } - private void importIteration( - UUID jobId, - AuthData importAuthData, - String jobIdPrefix, - int copyIteration, - DataModel exportedData) - throws CopyException { + private void importIteration( + UUID jobId, + AuthData importAuthData, + String jobIdPrefix, + int copyIteration, + DataModel exportedData) + throws CopyException { - monitor.debug( - () -> jobIdPrefix + "Starting import, copy iteration: " + copyIteration, - EventCode.COPIER_STARTED_IMPORT); + monitor.debug( + () -> jobIdPrefix + "Starting import, copy iteration: " + copyIteration, + EventCode.COPIER_STARTED_IMPORT); - CallableImporter callableImporter = - new CallableImporter( - importerProvider, - jobId, - idempotentImportExecutor, - importAuthData, - exportedData, - metricRecorder); - RetryingCallable retryingImporter = - new RetryingCallable<>( - callableImporter, - retryStrategyLibraryProvider.get(), - Clock.systemUTC(), - monitor, - JobMetadata.getDataType(), - JobMetadata.getImportService()); - boolean importSuccess = false; - Stopwatch importStopwatch = Stopwatch.createStarted(); - try { - ImportResult importResult = retryingImporter.call(); - importSuccess = importResult.getType() == ImportResult.ResultType.OK; - if (importSuccess) { + CallableImporter callableImporter = + new CallableImporter( + importerProvider, + jobId, + idempotentImportExecutor, + importAuthData, + exportedData, + metricRecorder); + RetryingCallable retryingImporter = + new RetryingCallable<>( + callableImporter, + retryStrategyLibraryProvider.get(), + Clock.systemUTC(), + monitor, + JobMetadata.getDataType(), + JobMetadata.getImportService()); + boolean importSuccess = false; + Stopwatch importStopwatch = Stopwatch.createStarted(); try { - jobStore.addCounts(jobId, importResult.getCounts().orElse(null)); - jobStore.addBytes(jobId, importResult.getBytes().orElse(null)); - } catch (IOException | DatastoreException e) { - monitor.debug(() -> jobIdPrefix + "Unable to add counts to job: ", e); + ImportResult importResult = retryingImporter.call(); + importSuccess = importResult.getType() == ImportResult.ResultType.OK; + if (importSuccess) { + try { + jobStore.addCounts(jobId, importResult.getCounts().orElse(null)); + jobStore.addBytes(jobId, importResult.getBytes().orElse(null)); + } catch (IOException | DatastoreException e) { + monitor.debug(() -> jobIdPrefix + "Unable to add counts to job: ", e); + } + } + monitor.debug( + () -> jobIdPrefix + "Finished import, copy iteration: " + copyIteration, + EventCode.COPIER_FINISHED_IMPORT); + } catch (RetryException e) { + if (!e.canSkip()) { + throw convertToCopyException(jobIdPrefix, "import", e); + } + } catch (RuntimeException e) { + throw convertToCopyException(jobIdPrefix, "import", e); + } finally { + metricRecorder.importPageFinished( + JobMetadata.getDataType(), new ServiceResult( + JobMetadata.getImportService(), + importSuccess, + importStopwatch.elapsed()) + ); } - } - monitor.debug( - () -> jobIdPrefix + "Finished import, copy iteration: " + copyIteration, - EventCode.COPIER_FINISHED_IMPORT); - } catch (RetryException e) { - if (!e.canSkip()) { - throw convertToCopyException(jobIdPrefix, "import", e); - } - } catch (RuntimeException e) { - throw convertToCopyException(jobIdPrefix, "import", e); - } finally { - metricRecorder.importPageFinished( - JobMetadata.getDataType(), - JobMetadata.getImportService(), - importSuccess, - importStopwatch.elapsed()); } - } - private void sizeCalculationIteration(UUID jobId, String jobIdPrefix, - DataModel exportedData) throws CopyException { - Collection items; - if (exportedData instanceof PhotosContainerResource) { - items = ((PhotosContainerResource) exportedData).getPhotos(); - } else if (exportedData instanceof VideosContainerResource) { - items = ((VideosContainerResource) exportedData).getVideos(); - } else if (exportedData instanceof MediaContainerResource) { - MediaContainerResource mcr = (MediaContainerResource) exportedData; - List list = new ArrayList<>(mcr.getVideos()); - list.addAll(mcr.getPhotos()); - items = list; - } else { - return; - } + private void sizeCalculationIteration(UUID jobId, String jobIdPrefix, + DataModel exportedData) throws CopyException { + Collection items; + if (exportedData instanceof PhotosContainerResource) { + items = ((PhotosContainerResource) exportedData).getPhotos(); + } else if (exportedData instanceof VideosContainerResource) { + items = ((VideosContainerResource) exportedData).getVideos(); + } else if (exportedData instanceof MediaContainerResource) { + MediaContainerResource mcr = (MediaContainerResource) exportedData; + List list = new ArrayList<>(mcr.getVideos()); + list.addAll(mcr.getPhotos()); + items = list; + } else { + return; + } - CallableSizeCalculator callableSizeCalculator = - new CallableSizeCalculator(jobId, new ConnectionProvider(jobStore), items); - try { - RetryingCallable> retryingImporter = - new RetryingCallable<>( - callableSizeCalculator, - retryStrategyLibraryProvider.get(), - Clock.systemUTC(), - monitor, - JobMetadata.getDataType(), - JobMetadata.getImportService()); - jobStore.addBytes(jobId, retryingImporter.call()); - } catch (RetryException | RuntimeException e) { - throw convertToCopyException(jobIdPrefix, "size estimation", e); + CallableSizeCalculator callableSizeCalculator = + new CallableSizeCalculator(jobId, new ConnectionProvider(jobStore), items); + try { + RetryingCallable> retryingImporter = + new RetryingCallable<>( + callableSizeCalculator, + retryStrategyLibraryProvider.get(), + Clock.systemUTC(), + monitor, + JobMetadata.getDataType(), + JobMetadata.getImportService()); + jobStore.addBytes(jobId, retryingImporter.call()); + } catch (RetryException | RuntimeException e) { + throw convertToCopyException(jobIdPrefix, "size estimation", e); + } } - } - private CopyException convertToCopyException(String jobIdPrefix, String suffix, Exception e) { - if (e.getClass() == RetryException.class - && CopyExceptionWithFailureReason.class.isAssignableFrom(e.getCause().getClass())) { - return (CopyExceptionWithFailureReason) e.getCause(); + private CopyException convertToCopyException(String jobIdPrefix, String suffix, Exception e) { + if (e.getClass() == RetryException.class + && CopyExceptionWithFailureReason.class.isAssignableFrom(e.getCause().getClass())) { + return (CopyExceptionWithFailureReason) e.getCause(); + } + return new CopyException(jobIdPrefix + "Error happened during " + suffix, e); } - return new CopyException(jobIdPrefix + "Error happened during " + suffix, e); - } } From af003674ba223710dafdc0e40b0d389d28a82d93 Mon Sep 17 00:00:00 2001 From: tschoemaker Date: Fri, 3 May 2024 16:48:50 +0200 Subject: [PATCH 2/2] refactored data clumps --- .../api/launcher/ServiceResult.java | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/ServiceResult.java b/portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/ServiceResult.java index e50bc8df8..426fcd3c4 100644 --- a/portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/ServiceResult.java +++ b/portability-api-launcher/src/main/java/org/datatransferproject/api/launcher/ServiceResult.java @@ -1,3 +1,18 @@ +/* + * Copyright 2019 The Data Transfer Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.datatransferproject.api.launcher; public class ServiceResult{ private java.lang.String service;