Add liquibase migration, fix remaining repository queries and smaller improvements.

Author: kinow

pom.xml

@@ -75,6 +75,16 @@
       <artifactId>hibernate-types-55</artifactId>
       <version>2.14.0</version>
     </dependency>
+    <!-- Liquibase; note that Flyway, while probably having more documentation and users,
+         changed its license model. Similarly to mongo, we prefer an open license that
+         won't lock users or developers from updating or modifying the code if necessary.
+         Hence our choice for liquibase. Feel free to suggest alternatives, but keep in
+         mind the license.
+    -->
+    <dependency>
+      <groupId>org.liquibase</groupId>
+      <artifactId>liquibase-core</artifactId>
+    </dependency>
     <!-- Postgres -->
     <dependency>
       <groupId>org.postgresql</groupId>

src/main/java/org/commonwl/view/WebConfig.java

@@ -83,6 +83,6 @@ public void configureContentNegotiation(ContentNegotiationConfigurer configurer)
 
     @Override
     public void addCorsMappings(CorsRegistry registry) {
-	registry.addMapping("/**").exposedHeaders("Location");  // .setMaxAge(Long.MAX_VALUE)
+        registry.addMapping("/**").exposedHeaders("Location");  // .setMaxAge(Long.MAX_VALUE)
     }
 }

src/main/java/org/commonwl/view/workflow/QueuedWorkflowRepository.java

@@ -1,7 +1,9 @@
 package org.commonwl.view.workflow;
 
 import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
 import org.springframework.data.jpa.repository.Query;
+import org.springframework.transaction.annotation.Transactional;
 
 import java.util.Date;
 import java.util.List;
@@ -20,16 +22,18 @@ public interface QueuedWorkflowRepository extends JpaRepository<QueuedWorkflow,
      * @param retrievedOn Date of when the queued workflow was retrieved
      * @return The number of queued workflows deleted
      */
-    @Query(value = "DELETE FROM queued_workflow q WHERE q.temp_representation ->> 'retrieved_on' <= ?1", nativeQuery = true)
-    Long deleteByTempRepresentation_RetrievedOnLessThanEqual(Date retrievedOn);
+    @Transactional
+    @Modifying
+    @Query(value = "DELETE FROM queued_workflow q WHERE q.temp_representation ->> 'retrievedOn' <= ?1", nativeQuery = true)
+    Integer deleteByTempRepresentation_RetrievedOnLessThanEqual(Date retrievedOn);
 
     /**
      * Finds and returns all queued workflows with date retrieved on older or equal to the Date argument passed.
      *
      * @param retrievedOn Details of where the queued workflow is from
      * @return A list of queued workflows
      */
-    @Query(value = "SELECT q.* FROM queued_workflow q WHERE q.temp_representation ->> 'retrieved_on' <= ?1", nativeQuery = true)
+    @Query(value = "SELECT q.* FROM queued_workflow q WHERE q.temp_representation ->> 'retrievedOn' <= ?1", nativeQuery = true)
     List<QueuedWorkflow> findByTempRepresentation_RetrievedOnLessThanEqual(Date retrievedOn);
 
 }

src/main/java/org/commonwl/view/workflow/WorkflowController.java

@@ -20,21 +20,16 @@
 package org.commonwl.view.workflow;
 
 import java.io.File;
-import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.io.InputStream;
-import java.security.DigestInputStream;
-import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.util.List;
 
 import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import javax.validation.Valid;
 
-import org.apache.commons.codec.binary.Hex;
-import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang.StringUtils;
 import org.commonwl.view.WebConfig;
 import org.commonwl.view.cwl.CWLService;
@@ -236,7 +231,7 @@ public Resource getROBundle(@PathVariable("domain") String domain,
                                           @PathVariable("repoName") String repoName,
                                           @PathVariable("branch") String branch,
                                           HttpServletRequest request,
-                                          HttpServletResponse response) throws IOException {
+                                          HttpServletResponse response) {
         String path = (String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE);
         path = extractPath(path, 7);
         GitDetails gitDetails = getGitDetails(domain, owner, repoName, branch, path);
@@ -254,7 +249,7 @@ public Resource getROBundle(@PathVariable("domain") String domain,
     @ResponseBody
     public Resource getROBundleGeneric(@PathVariable("branch") String branch,
                                                  HttpServletRequest request,
-                                                 HttpServletResponse response) throws IOException {
+                                                 HttpServletResponse response) {
         String path = (String) request.getAttribute(HandlerMapping.PATH_WITHIN_HANDLER_MAPPING_ATTRIBUTE);
         GitDetails gitDetails = getGitDetails(10, path, branch);
         File bundleDownload = workflowService.getROBundle(gitDetails);
@@ -584,7 +579,7 @@ private ModelAndView getWorkflow(GitDetails gitDetails, RedirectAttributes redir
     }
 
     private Resource getGraphFromInputStream(InputStream in, String format)
-            throws IOException, NoSuchAlgorithmException {
+            throws IOException {
         Workflow workflow = cwlService.parseWorkflowNative(in, null, "workflow"); // first workflow will do
         InputStream out = graphVizService.getGraphStream(workflow.getVisualisationDot(), format);
         return new InputStreamResource(out);

src/main/java/org/commonwl/view/workflow/WorkflowOverview.java

@@ -62,4 +62,4 @@ public boolean equals(Object o) {
     public int hashCode() {
         return Objects.hash(fileName, label, doc);
     }
-}
+}

src/main/java/org/commonwl/view/workflow/WorkflowPermalinkController.java

@@ -271,7 +271,8 @@ public Resource getROBundle(@PathVariable("commitid") String commitId,
      *            The commit ID of the repository
      * @param request
      *            The HttpServletRequest from the controller to extract path
-     * @param part2
+     * @param part
+     *            The workflow part
      * @throws WorkflowNotFoundException
      *             If workflow could not be found (404)
      * @throws MultipleWorkflowsException

src/main/java/org/commonwl/view/workflow/WorkflowRepository.java

@@ -48,7 +48,7 @@ public interface WorkflowRepository extends JpaRepository<Workflow, String> {
      * @param path The path to the workflow within the repository
      * @return The workflow model
      */
-    @Query(value = "SELECT w.* FROM workflow w WHERE w.lastCommit = ?1 AND w.retrievedFrom ->> 'path' = ?2", nativeQuery = true)
+    @Query(value = "SELECT w.* FROM workflow w WHERE w.last_commit = ?1 AND w.retrieved_from ->> 'path' = ?2", nativeQuery = true)
     List<Workflow> findByCommitAndPath(String commitId, String path);
 
     /**

src/main/java/org/commonwl/view/workflow/WorkflowService.java

@@ -47,7 +47,6 @@
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Value;
-import org.springframework.core.io.FileSystemResource;
 import org.springframework.core.io.PathResource;
 import org.springframework.data.domain.Page;
 import org.springframework.data.domain.Pageable;
@@ -446,7 +445,7 @@ public Workflow findByCommitAndPath(String commitID, String path, Optional<Strin
      * @param gitDetails The Git details of the workflow
      * @return A FileSystemResource representing the graph
      * @throws WorkflowNotFoundException Error getting the workflow or format
-     * @throws IOException 
+     * @throws IOException Error reading the workflow files
      */
     public PathResource getWorkflowGraph(String format, GitDetails gitDetails)
             throws WorkflowNotFoundException, IOException {

src/main/resources/application.properties

@@ -50,10 +50,11 @@ spring.datasource.password=sa
 spring.sql.init.mode=always
 spring.sql.init.platform=postgres
 
+spring.jpa.generate-ddl=false
 spring.jpa.open-in-view=false
 spring.jpa.show-sql=false
 spring.jpa.database-platform=org.hibernate.dialect.PostgreSQLDialect
-spring.jpa.hibernate.ddl-auto=none
+spring.jpa.hibernate.ddl-auto=validate
 spring.jpa.properties.hibernate.jdbc.lob.non_contextual_creation=true
 
 #=======================
@@ -72,4 +73,10 @@ sparql.endpoint = http://localhost:3030/cwlviewer/
 cron.deleteOldQueuedWorkflows = 0 0 * * * ?
 
 # Age limit for queued workflows in hours.
-queuedWorkflowAgeLimitHours = 24
+queuedWorkflowAgeLimitHours = 24
+
+#=======================
+# DB migrations
+#=======================
+spring.liquibase.enabled=true
+spring.liquibase.change-log=classpath:db/changelog/db.changelog.xml

src/main/resources/db/changelog/db.changelog.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<databaseChangeLog
+  xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xmlns:pro="http://www.liquibase.org/xml/ns/pro"
+  xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
+  http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.1.xsd
+  http://www.liquibase.org/xml/ns/pro
+  http://www.liquibase.org/xml/ns/pro/liquibase-pro-4.1.xsd">
+  <includeAll path="db/changelog/migrations/"/>
+</databaseChangeLog>
+

src/main/resources/db/changelog/migrations/00001_schema_ddl.sql

@@ -0,0 +1,42 @@
+--liquibase formatted sql
+
+--changeset kinow:create-workflow-table
+create table if not exists workflow
+(
+    id                varchar(36) not null
+        primary key,
+    cwltool_version   text,
+    doc               text,
+    docker_link       text,
+    inputs            jsonb,
+    label             text,
+    last_commit       text,
+    license_link      text,
+    outputs           jsonb,
+    retrieved_from    jsonb
+        constraint unique_workflow_retrieved_from
+            unique,
+    retrieved_on      timestamp,
+    ro_bundle_path    text,
+    steps             jsonb,
+    visualisation_dot text
+);
+--rollback drop table workflow;
+
+--changeset kinow:create-idx_workflow_retrieved_on-index
+create index if not exists idx_workflow_retrieved_on
+    on workflow (retrieved_on);
+--rollback drop index idx_workflow_retrieved_on;
+
+--changeset kinow:create-queued_workflow-table
+create table if not exists queued_workflow
+(
+    id                  varchar(36) not null
+        primary key,
+    cwltool_status      jsonb,
+    cwltool_version     text,
+    message             text,
+    temp_representation jsonb,
+    workflow_list       jsonb
+);
+--rollback drop table queued_workflow;

src/main/resources/logback.xml

@@ -54,4 +54,4 @@
   </logger>
   -->
 
-</configuration>
+</configuration>

src/test/resources/it-application.properties

@@ -8,4 +8,4 @@ spring.jpa.properties.hibernate.format_sql=true
 logging.level.org.hibernate.type.descriptor.sql=trace
 
 logging.level.org.hibernate.SQL=DEBUG
-logging.level.org.hibernate.type.descriptor.sql.BasicBinder=TRACE
+logging.level.org.hibernate.type.descriptor.sql.BasicBinder=TRACE