Skip to content

Refactored data clumps with the help of LLMs (research project) #1355

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 2 commits into from
Closed

Refactored data clumps with the help of LLMs (research project) #1355

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
89729a9
with errors
compf May 3, 2024
af00367
refactored data clumps
compf May 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
649 changes: 316 additions & 333 deletions ...e/src/main/java/org/datatransferproject/cloud/google/GoogleDtpInternalMetricRecorder.java
View file
Open in desktop

Large diffs are not rendered by default.

253 changes: 123 additions & 130 deletions ...n/java/org/datatransferproject/cloud/microsoft/cosmos/AzureDtpInternalMetricRecorder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,143 +15,136 @@
*/
package org.datatransferproject.cloud.microsoft.cosmos;

import static java.lang.String.format;

import org.datatransferproject.api.launcher.DtpInternalMetricRecorder;
import org.datatransferproject.api.launcher.Monitor;
import org.datatransferproject.api.launcher.ServiceResult;
import org.datatransferproject.types.common.models.DataVertical;

import java.time.Duration;
import org.datatransferproject.types.common.models.DataVertical;

import static java.lang.String.format;

/**
* A placeholder {@link DtpInternalMetricRecorder} that simply logs metrics
* to the default monitor.
* **/
**/
class AzureDtpInternalMetricRecorder implements DtpInternalMetricRecorder {
private final Monitor monitor;

// TODO: Replace with a Azure Monitor based implementation.
AzureDtpInternalMetricRecorder(Monitor monitor) {
this.monitor = monitor;
}

@Override
public void startedJob(DataVertical dataType, String exportService, String importService) {
monitor.debug(
() ->
format(
"Metric: StartedJob, data type: %s, from: %s, to: %s",
dataType, exportService, importService));
}

@Override
public void exportPageAttemptFinished(
DataVertical dataType,
String service,
boolean success,
Duration duration) {
monitor.debug(
() ->
format(
"Metric: exportPageAttemptFinished, data type: %s, service: %s, "
+ "success: %s, duration: %s",
dataType, service, success, duration));
}

@Override
public void exportPageFinished(
DataVertical dataType,
String service,
boolean success,
Duration duration) {
monitor.debug(
() ->
format(
"Metric: exportPageFinished, data type: %s, service: %s, success: %s, duration: %s",
dataType, service, success, duration));
}

@Override
public void importPageAttemptFinished(
DataVertical dataType,
String service,
boolean success,
Duration duration) {
monitor.debug(
() ->
format(
"Metric: importPageAttemptFinished, data type: %s, service: %s,"
+ "success: %s, duration: %s",
dataType, service, success, duration));
}

@Override
public void importPageFinished(
DataVertical dataType,
String service,
boolean success,
Duration duration) {
monitor.debug(
() ->
format(
"Metric: importPageFinished, data type: %s, service: %s, success: %s, duration: %s",
dataType, service, success, duration));
}

@Override
public void finishedJob(
DataVertical dataType,
String exportService,
String importService,
boolean success,
Duration duration) {
monitor.debug(
() ->
format(
"Metric: finishedJob, data type: %s, from: %s, to: %s, success: %s, duration: %s",
dataType, exportService, importService, success, duration));
}

@Override
public void cancelledJob(DataVertical dataType, String exportService, String importService, Duration duration) {
monitor.debug(
() ->
format("Metric: cancelledJob, data type: %s, from: %s, to: %s, duration: %s",
dataType, exportService, importService, duration));

}

@Override
public void recordGenericMetric(DataVertical dataType, String service, String tag) {
monitor.debug(
() ->
format("Metric: Generic, data type: %s, service: %s, tag: %s", dataType, service, tag));
}

@Override
public void recordGenericMetric(DataVertical dataType, String service, String tag, boolean bool) {
monitor.debug(
() ->
format(
"Metric: Generic, data type: %s, service: %s, tag: %s, value: %s",
dataType, service, tag, bool));
}

@Override
public void recordGenericMetric(DataVertical dataType, String service, String tag, Duration duration) {
monitor.debug(
() ->
format(
"Metric: Generic, data type: %s, service: %s, tag: %s, duration: %s",
dataType, service, tag, duration));
}

@Override
public void recordGenericMetric(DataVertical dataType, String service, String tag, int value) {
monitor.debug(
() ->
format(
"Metric: Generic, data type: %s, service: %s, tag: %s, value: %s",
dataType, service, tag, value));
}
private final Monitor monitor;

// TODO: Replace with a Azure Monitor based implementation.
AzureDtpInternalMetricRecorder(Monitor monitor) {
this.monitor = monitor;
}

@Override
public void startedJob(DataVertical dataType, String exportService, String importService) {
monitor.debug(
() ->
format(
"Metric: StartedJob, data type: %s, from: %s, to: %s",
dataType, exportService, importService));
}

@Override
public void exportPageAttemptFinished(
DataVertical dataType,
ServiceResult serviceResult) {
monitor.debug(
() ->
format(
"Metric: exportPageAttemptFinished, data type: %s, service: %s, "
+ "success: %s, duration: %s",
dataType, serviceResult.getService(), serviceResult.getSuccess(), serviceResult.getDuration()));
}

@Override
public void exportPageFinished(
DataVertical dataType,
ServiceResult serviceResult) {
monitor.debug(
() ->
format(
"Metric: exportPageFinished, data type: %s, service: %s, success: %s, duration: %s",
dataType, serviceResult.getService(), serviceResult.getSuccess(), serviceResult.getDuration()));
}

@Override
public void importPageAttemptFinished(
DataVertical dataType,
ServiceResult serviceResult) {
monitor.debug(
() ->
format(
"Metric: importPageAttemptFinished, data type: %s, service: %s,"
+ "success: %s, duration: %s",
dataType, serviceResult.getService(), serviceResult.getSuccess(), serviceResult.getDuration()));
}

@Override
public void importPageFinished(
DataVertical dataType,
ServiceResult serviceResult) {
monitor.debug(
() ->
format(
"Metric: importPageFinished, data type: %s, service: %s, success: %s, duration: %s",
dataType, serviceResult.getService(), serviceResult.getSuccess(), serviceResult.getDuration()));
}

@Override
public void finishedJob(
DataVertical dataType,
String exportService,
String importService,
boolean success,
Duration duration) {
monitor.debug(
() ->
format(
"Metric: finishedJob, data type: %s, from: %s, to: %s, success: %s, duration: %s",
dataType, exportService, importService, success, duration));
}

@Override
public void cancelledJob(DataVertical dataType, String exportService, String importService, Duration duration) {
monitor.debug(
() ->
format("Metric: cancelledJob, data type: %s, from: %s, to: %s, duration: %s",
dataType, exportService, importService, duration));

}

@Override
public void recordGenericMetric(DataVertical dataType, String service, String tag) {
monitor.debug(
() ->
format("Metric: Generic, data type: %s, service: %s, tag: %s", dataType, service, tag));
}

@Override
public void recordGenericMetric(DataVertical dataType, String service, String tag, boolean bool) {
monitor.debug(
() ->
format(
"Metric: Generic, data type: %s, service: %s, tag: %s, value: %s",
dataType, service, tag, bool));
}

@Override
public void recordGenericMetric(DataVertical dataType, String service, String tag, Duration duration) {
monitor.debug(
() ->
format(
"Metric: Generic, data type: %s, service: %s, tag: %s, duration: %s",
dataType, service, tag, duration));
}

@Override
public void recordGenericMetric(DataVertical dataType, String service, String tag, int value) {
monitor.debug(
() ->
format(
"Metric: Generic, data type: %s, service: %s, tag: %s, value: %s",
dataType, service, tag, value));
}
}
100 changes: 62 additions & 38 deletions ...auncher/src/main/java/org/datatransferproject/api/launcher/DtpInternalMetricRecorder.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,10 @@
*/
package org.datatransferproject.api.launcher;

import java.time.Duration;
import org.datatransferproject.types.common.models.DataVertical;

import java.time.Duration;

/**
* Interface to log metrics about a DTP job.
*
Expand All @@ -28,47 +29,70 @@
* platform.
*/
public interface DtpInternalMetricRecorder {
// Metrics related to DTP internals
// Metrics related to DTP internals

/**
* A DTP job started.
**/
void startedJob(DataVertical dataType, String exportService, String importService);

/**
* A DTP job finished
**/
void finishedJob(
DataVertical dataType,
String exportService,
String importService,
boolean success,
Duration duration);

/**
* A DTP job cancelled
**/
void cancelledJob(
DataVertical dataType,
String exportService,
String importService,
Duration duration);

/**
* An single attempt to export a page of data finished.
*
* @param serviceResult The ServiceResult object
*/
void exportPageAttemptFinished(
DataVertical dataType,
ServiceResult serviceResult);

/**
* An attempt to export a page of data finished including all retires.
*
* @param serviceResult The ServiceResult object
*/
void exportPageFinished(DataVertical dataType, ServiceResult serviceResult);

/** A DTP job started. **/
void startedJob(DataVertical dataType, String exportService, String importService);
/** A DTP job finished **/
void finishedJob(
DataVertical dataType,
String exportService,
String importService,
boolean success,
Duration duration);
/** A DTP job cancelled **/
void cancelledJob(
DataVertical dataType,
String exportService,
String importService,
Duration duration);
/**
* An single attempt to import a page of data finished.
*
* @param serviceResult The ServiceResult object
*/
void importPageAttemptFinished(
DataVertical dataType,
ServiceResult serviceResult);

/** An single attempt to export a page of data finished. **/
void exportPageAttemptFinished(
DataVertical dataType,
String service,
boolean success,
Duration duration);
/**
* An attempt to import a page of data finished including all retires.
*
* @param serviceResult The ServiceResult object
**/
void importPageFinished(DataVertical dataType, ServiceResult serviceResult);

/** An attempt to export a page of data finished including all retires. **/
void exportPageFinished(DataVertical dataType, String service, boolean success, Duration duration);
// Metrics from {@link MetricRecorder}
void recordGenericMetric(DataVertical dataType, String service, String tag);

/** An single attempt to import a page of data finished. **/
void importPageAttemptFinished(
DataVertical dataType,
String service,
boolean success,
Duration duration);
void recordGenericMetric(DataVertical dataType, String service, String tag, boolean bool);

/** An attempt to import a page of data finished including all retires. **/
void importPageFinished(DataVertical dataType, String service, boolean success, Duration duration);
void recordGenericMetric(DataVertical dataType, String service, String tag, Duration duration);

// Metrics from {@link MetricRecorder}
void recordGenericMetric(DataVertical dataType, String service, String tag);
void recordGenericMetric(DataVertical dataType, String service, String tag, boolean bool);
void recordGenericMetric(DataVertical dataType, String service, String tag, Duration duration);
void recordGenericMetric(DataVertical dataType, String service, String tag, int value);
void recordGenericMetric(DataVertical dataType, String service, String tag, int value);
}
Loading