Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
9ce3305
tidy up agent scripts, check exit status directly instead of indirectly
Ushcode Mar 5, 2024
15a922a
move SQL to resources directory
Ushcode Mar 28, 2024
bc65867
tidy up java source code
Ushcode Mar 28, 2024
61369da
skip test that won't pass without spinning up in a stack
Ushcode Mar 28, 2024
4d2f409
increment version and update Dockerfile to allow different war versions
Ushcode Mar 28, 2024
fc2849b
CReDo isochrone config
Ushcode Mar 28, 2024
7648e49
Revert "CReDo isochrone config"
Ushcode Mar 28, 2024
e948aa9
CReDo isochrone config
Ushcode Mar 28, 2024
c7e7404
Merge branch 'dev-op-isochroneagent' of https://github.com/cambridge-…
Ushcode Mar 28, 2024
e1d1055
stack.sh permission +x
Ushcode Mar 28, 2024
b67e5ab
read sql queries as one string
Ushcode Mar 28, 2024
31c2671
fix queries and add debug file
Ushcode Mar 28, 2024
a2d408d
Merge branch 'dev-op-isochroneagent' of https://github.com/cambridge-…
Ushcode Mar 28, 2024
2d4e3c4
separate isochrone creation
Ushcode Mar 28, 2024
cf6bdb8
resource SQL for PopulationMapper.java
Ushcode Apr 23, 2024
f5237d7
Merge branch 'dev-op-routing-agents-hotfix' into dev-op-isochroneagent
Ushcode May 2, 2024
c48bc12
Merge branch 'main' into dev-op-isochroneagent
Ushcode May 2, 2024
b2121d9
dev-op-isochroneagent: update and format docs
Ushcode Aug 7, 2024
17aa2d8
dev-op-isochroneagent: rename start script
Ushcode Aug 7, 2024
05b121d
dev-op-isochroneagent: update deps
Ushcode Aug 7, 2024
894b56c
dev-op-isochroneagent: tidy pom
Ushcode Aug 9, 2024
c418f7c
dev-op-isochroneagent: Merge branch 'main' into dev-op-isochroneagent
Ushcode Aug 9, 2024
1efe6a0
dev-op-isochroneagent: rm deprecated docker compose version
Ushcode Aug 9, 2024
56cf93f
dev-op-isochroneagent: rm blazegraph endpoint config bits since they'…
Ushcode Aug 9, 2024
617a209
dev-op-isochroneagent: standardise var names, add a private constructor
Ushcode Aug 9, 2024
0320e5c
dev-op-isochroneagent: standardise some var names, find replace on th…
Ushcode Aug 9, 2024
2f7e199
dev-op-isochroneagent: make big query ready for find replace
Ushcode Aug 9, 2024
6687711
dev-op-isochroneagent: accept some linting suggestions on IsochroneAg…
Ushcode Aug 9, 2024
0e1d09d
dev-op-isochroneagent: credo config.props
Ushcode Aug 9, 2024
2620a25
dev-op-isochroneagent: ontop service to substation query
Ushcode Aug 9, 2024
90c0dcc
dev-op-isochroneagent: rm docker-compose version
Ushcode Aug 9, 2024
0300d51
dev-op-isochroneagent: revert some pom changes
Ushcode Aug 9, 2024
0242f97
dev-op-isochroneagent: remove public test qualifier, add teardown of …
Ushcode Aug 12, 2024
d1fa5c4
dev-op-isochroneagent: tidy isochroneagentTest
Ushcode Aug 12, 2024
7b7bee2
dev-op-isochroneagent: tidy isochroneagentTest
Ushcode Aug 12, 2024
51c5552
Merge branch 'main' into dev-op-isochroneagent
Ushcode Dec 5, 2024
575052e
Merge remote-tracking branch 'origin/main' into dev-op-isochroneagent
Ushcode Dec 5, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions Agents/IsochroneAgent/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
# First stage: build war file
#==================================================================================================
FROM maven:3.6-openjdk-11-slim as builder
FROM maven:3.6-openjdk-11-slim AS builder

# Copy all files into root's home, including the source, pom file, ./m2 directory and credentials
ADD . /root
Expand All @@ -17,10 +17,9 @@ WORKDIR /root/isochroneagent
RUN --mount=type=cache,target=/root/.m2/repository mvn package

#==================================================================================================

# Second stage: copy the downloaded dependency into a new image and build into an app
#==================================================================================================
FROM tomcat:9.0 as agent
FROM tomcat:9.0 AS agent

WORKDIR /root/isochroneagent

Expand Down
7 changes: 7 additions & 0 deletions Agents/IsochroneAgent/docker-compose-debug.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
version: '3.8'
services:
stack-manager:
environment:
- "JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=*:5005"
ports:
- "${DEBUG_PORT}:5005"
1 change: 0 additions & 1 deletion Agents/IsochroneAgent/docker-compose-stack.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
version: '3.8'
services:
isochroneagent:
deploy:
Expand Down
2 changes: 1 addition & 1 deletion Agents/IsochroneAgent/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ version: "3.8"

services:
isochroneagent:
image: ghcr.io/cambridge-cares/isochroneagent:1.5.0
image: ghcr.io/cambridge-cares/isochroneagent:1.5.0-op-SNAPSHOT
build: .
ports:
- "10105:8080"
Expand Down
2 changes: 1 addition & 1 deletion Agents/IsochroneAgent/inputs/config.properties
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#DbName
db.name=postgres
db.name=credo_roads

#Segmentization length
segmentization_length= 0.000045
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
PREFIX obe: <https://www.theworldavatar.com/kg/ontobuiltenv/>
PREFIX rdf: <http://www.w3.org/1999/02/22-rdf-syntax-ns#>
PREFIX geo: <http://www.opengis.net/ont/geosparql#>
PREFIX geof: <http://www.opengis.net/def/function/geosparql/>
PREFIX uom: <http://www.opengis.net/def/uom/OGC/1.0/>
PREFIX ukpn: <http://theworldavatar.com/ontology/ontocredo/ontoukpn.owl#>

SELECT DISTINCT ?poi_iri ?poi_type ?geometry
WHERE {
{
SERVICE [ONTOP] {
SELECT DISTINCT * WHERE {
?sub a ukpn:PrimarySubStation ;
geo:hasDefaultGeometry/geo:asWKT ?geometry

BIND(?sub AS ?poi_iri)
BIND(ukpn:PrimarySubStation AS ?poi_type)
}
LIMIT 10
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
SELECT gid as id, source, target, cost_s as cost FROM routing_ways_segment WHERE tag_id IN (100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 121, 123, 124, 125, 401)
13 changes: 5 additions & 8 deletions Agents/IsochroneAgent/isochroneagent/pom.xml
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>uk.ac.cam.cares.jps</groupId>
<artifactId>isochroneagent</artifactId>
<version>1.5.0</version>
<version>1.5.0-op-SNAPSHOT</version>
<packaging>war</packaging>

<properties>
Expand Down Expand Up @@ -45,14 +45,12 @@
<id>prod-profile</id>
<properties>
<!-- Set property to download production logging config -->
<log.artifact>java-logging-prod</log.artifact>
</properties>
</profile>
</profiles>

<!-- Build settings -->
<build>

<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
Expand All @@ -64,7 +62,7 @@
</configuration>
</plugin>

<!-- Allows maven executions -->
<!-- Allows maven executions -->
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>exec-maven-plugin</artifactId>
Expand Down Expand Up @@ -122,7 +120,7 @@
<dependency>
<groupId>org.json</groupId>
<artifactId>json</artifactId>
<version>20180813</version>
<version>20240303</version>
<type>jar</type>
</dependency>
<!-- Logging, versions pulled from parent -->
Expand Down Expand Up @@ -162,7 +160,6 @@
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.5.4</version>
</dependency>
<!-- for interacting with the stack -->
<dependency>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,12 +6,9 @@

public class EndpointConfig {
private PostGISEndpointConfig postGISEndpointConfig;
private String dburl;
private String dbUser;
private String dbPassword;
private String kgurl;
private String kguser;
private String kgpassword;


public EndpointConfig() {
Expand All @@ -23,10 +20,6 @@ public EndpointConfig() {
BlazegraphEndpointConfig blazegraphEndpointConfig = containerClient.readEndpointConfig("blazegraph",
BlazegraphEndpointConfig.class);
this.kgurl = blazegraphEndpointConfig.getUrl("kb");
this.kguser = blazegraphEndpointConfig.getUsername();
this.kgpassword = blazegraphEndpointConfig.getPassword();


}

public String getDbUrl(String dbName) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,14 @@

public class FileReader {

// Private constructor to hide the implicit public one
private FileReader() {
throw new UnsupportedOperationException("This is a utility class and cannot be instantiated");
}

/**
* Read input files
*
* @param path
* @return
* @throws FileNotFoundException
Expand All @@ -26,14 +32,15 @@ public static InputStream getStream(String path) throws FileNotFoundException {
return new FileInputStream(path);
}


/**
* Read Point of Interest (POI) files from directory and parse into Map, allows multiple SPARQL files.
* @param POI_PATH Path for directory containing POI SPARQL queries
* Read Point of Interest (POI) files from directory and parse into Map, allows
* multiple SPARQL files.
*
* @param poiPath Path for directory containing POI SPARQL queries
* @return
*/
public static Map readPOIsparql(Path POI_PATH) {
try (Stream<Path> files = Files.list(POI_PATH)) {
public static Map<String, String> readPOIsparql(Path poiPath) {
try (Stream<Path> files = Files.list(poiPath)) {
// Find all available SPARQL files
Map<String, String> sparqlFiles = files
.filter(Files::isRegularFile)
Expand All @@ -50,24 +57,25 @@ public static Map readPOIsparql(Path POI_PATH) {
}));

// Process each SPARQL file
sparqlFiles.forEach((fileName, filePath) -> {
// Process the SPARQL file here
// You can replace the following line with your processing logic
System.out.println("Processing SPARQL file: " + filePath);
});
sparqlFiles.forEach((fileName, filePath) ->
// Process the SPARQL file here
// You can replace the following line with your processing logic
System.out.println("Processing SPARQL file: " + filePath));
return sparqlFiles;
} catch (IOException ex) {
throw new RuntimeException("Failed to read files from the directory.", ex);
throw new RuntimeException("Failed to read SPARQL files from" + poiPath, ex);
}
}

/**
* Read EDGESTABLE SQL files from directory and parse into Map, allows multiple .SQL files.
* @param EDGESTABLESQL_PATH Path for directory containing .SQL
* Read EDGESTABLE SQL files from directory and parse into Map, allows multiple
* .SQL files.
*
* @param edgesTableSQLPath Path for directory containing .SQL
* @return
*/
public static Map readEdgesTableSQL(Path EDGESTABLESQL_PATH) {
try (Stream<Path> files = Files.list(EDGESTABLESQL_PATH)) {
public static Map<String, String> readEdgesTableSQL(Path edgesTableSQLPath) {
try (Stream<Path> files = Files.list(edgesTableSQLPath)) {
// Find all available EdgesTableSQL files
Map<String, String> edgesTableSQLFiles = files
.filter(Files::isRegularFile)
Expand All @@ -84,33 +92,36 @@ public static Map readEdgesTableSQL(Path EDGESTABLESQL_PATH) {
}));

// Process each EdgesTableSQL file
edgesTableSQLFiles.forEach((fileName, filePath) -> {
edgesTableSQLFiles.forEach((fileName, filePath) ->
// Process the EdgesTableSQL file here
// You can replace the following line with your processing logic
System.out.println("Processing EdgesTableSQL file: " + filePath);
});
System.out.println("Processing EdgesTableSQL file: " + filePath)
);

return edgesTableSQLFiles;
} catch (IOException ex) {
throw new RuntimeException("Failed to read files from the directory.", ex);
throw new RuntimeException("Failed to read SQL files from" + edgesTableSQLPath, ex);
}
}

/** Retrieve POI locations from knowledge graph by executing the input SPARQL queries.
/**
* Retrieve POI locations from knowledge graph by executing the input SPARQL
* queries.
*
* @param storeClient
* @param POImap
* @param poiMap
* @return
*/
public static JSONArray getPOILocation(RemoteStoreClient storeClient, Map<String, String> POImap)
{
public static JSONArray getPOILocation(RemoteStoreClient storeClient, Map<String, String> poiMap) {
JSONArray cumulativePOI = new JSONArray();
for (Map.Entry<String, String> entry : POImap.entrySet()) {
for (Map.Entry<String, String> entry : poiMap.entrySet()) {
String value = entry.getValue();
JSONArray POI = storeClient.executeQuery(value);
JSONArray poi = storeClient.executeQuery(value);

// Iterate through the POIs in this iteration and add them to the cumulative array
for (int i = 0; i < POI.length(); i++) {
cumulativePOI.put(POI.get(i));
// Iterate through the POIs in this iteration and add them to the cumulative
// array
for (int i = 0; i < poi.length(); i++) {
cumulativePOI.put(poi.get(i));
}
}
return cumulativePOI;
Expand Down
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
package uk.ac.cam.cares.jps.agent.isochroneagent;

import javax.servlet.annotation.WebServlet;
import javax.ws.rs.BadRequestException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
Expand All @@ -18,15 +14,17 @@

import org.json.JSONArray;
import org.json.JSONObject;
import uk.ac.cam.cares.jps.base.agent.JPSAgent;
import uk.ac.cam.cares.jps.base.exception.JPSRuntimeException;
import uk.ac.cam.cares.jps.base.query.RemoteRDBStoreClient;
import uk.ac.cam.cares.jps.base.query.RemoteStoreClient;

import com.cmclinnovations.stack.clients.geoserver.GeoServerClient;
import com.cmclinnovations.stack.clients.geoserver.GeoServerVectorSettings;
import com.cmclinnovations.stack.clients.geoserver.UpdatedGSVirtualTableEncoder;
import com.cmclinnovations.stack.clients.ontop.OntopClient;

import uk.ac.cam.cares.jps.base.agent.JPSAgent;
import uk.ac.cam.cares.jps.base.exception.JPSRuntimeException;
import uk.ac.cam.cares.jps.base.query.RemoteRDBStoreClient;
import uk.ac.cam.cares.jps.base.query.RemoteStoreClient;

@WebServlet(urlPatterns = "/update")

public class IsochroneAgent extends JPSAgent {
Expand All @@ -36,15 +34,15 @@ public class IsochroneAgent extends JPSAgent {
private static int timeInterval;
private static final String PROPETIES_PATH = "/inputs/config.properties";
private static final Path obdaFile = Path.of("/inputs/isochrone.obda");
private final String FUNCTION_KEY = "function";
private final String TIMETHRESHOLD_KEY = "timethreshold";
private final String TIMEINTERVAL_KEY = "timeinterval";
private static final String FUNCTION_KEY = "function";
private static final String TIMETHRESHOLD_KEY = "timethreshold";
private static final String TIMEINTERVAL_KEY = "timeinterval";

private static final Logger LOGGER = LogManager.getLogger(IsochroneAgent.class);

private EndpointConfig endpointConfig = new EndpointConfig();
private final EndpointConfig endpointConfig = new EndpointConfig();
private String dbName;
private static String populationTables;
private String populationTables;
private String kgEndpoint;
private RemoteStoreClient storeClient;
private RemoteRDBStoreClient remoteRDBStoreClient;
Expand All @@ -63,6 +61,7 @@ public class IsochroneAgent extends JPSAgent {
/**
* Initialise agent
*/
@Override
public void init() {
readConfig();

Expand All @@ -84,7 +83,7 @@ public void init() {
* Read configuration settings from config.properties
*/
public void readConfig() {
try (InputStream input = FileReader.getStream(PROPETIES_PATH)) {
try (InputStream input = FileReader.getStream(PROPERTIES_PATH)) {
Properties prop = new Properties();
prop.load(input);
this.dbName = prop.getProperty("db.name");
Expand Down Expand Up @@ -148,6 +147,9 @@ public void readConfig() {
*/
@Override
public JSONObject processRequestParameters(JSONObject requestParams) {
int timeInterval;
int timeThreshold;
String isochroneFunction = null;

if (!validateInput(requestParams)) {
throw new JPSRuntimeException("Unable to validate request sent to the agent.");
Expand Down Expand Up @@ -257,6 +259,15 @@ public JSONObject processRequestParameters(JSONObject requestParams) {
return response;
}

private void tryToUpdateMapping() {
try {
OntopClient ontopClient = OntopClient.getInstance();
ontopClient.updateOBDA(obdaFile);
} catch (Exception e) {
System.out.println("Could not retrieve isochrone .obda file.");
}
}

/**
* Check if the JSONObject in the processRequestParameters inputs are correct or
* missing.
Expand Down
Loading