Skip to content

Commit 786cb15

Browse files
nvnieuwknvnieuwk
authored
Merge pull request #85 from nextflow-io/2.3.0dev
Release PR 2.3.0
2 parents 803c1ae + d48eac9 commit 786cb15

16 files changed

+256
-33
lines changed

‎CHANGELOG.md

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
# nextflow-io/nf-schema: Changelog
22

3+
# Version 2.3.0 - Hakodate
4+
5+
## Bug fixes
6+
7+
1. The help message will now also be printed out when no functions of the plugin get included in the pipeline.
8+
2. JSON and YAML files that are not a list of values should now also be validated correctly. (Mind that samplesheets always have to be a list of values to work with `samplesheetToList`)
9+
310
# Version 2.2.1
411

512
## Bug fixes

‎plugins/nf-schema/src/main/nextflow/validation/CustomEvaluators/SchemaEvaluator.groovy

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,11 +50,11 @@ class SchemaEvaluator implements Evaluator {
5050
log.debug("Started validating ${file.toString()}")
5151

5252
def String schemaFull = Utils.getSchemaPath(this.baseDir, this.schema)
53-
def JSONArray arrayJSON = Utils.fileToJsonArray(file, Path.of(schemaFull))
53+
def Object json = Utils.fileToJson(file, Path.of(schemaFull))
5454
def String schemaContents = Files.readString( Path.of(schemaFull) )
5555
def validator = new JsonSchemaValidator(config)
5656

57-
def List<String> validationErrors = validator.validate(arrayJSON, schemaContents)
57+
def List<String> validationErrors = validator.validate(json, schemaContents)
5858
if (validationErrors) {
5959
def List<String> errors = ["Validation of file failed:"] + validationErrors.collect { "\t${it}" as String}
6060
return Evaluator.Result.failure(errors.join("\n"))

‎plugins/nf-schema/src/main/nextflow/validation/SamplesheetConverter.groovy

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,14 +80,21 @@ class SamplesheetConverter {
8080
throw new SchemaValidationException(msg)
8181
}
8282

83+
def LinkedHashMap schemaMap = new JsonSlurper().parseText(schemaFile.text) as LinkedHashMap
84+
def List<String> schemaKeys = schemaMap.keySet() as List<String>
85+
if(schemaKeys.contains("properties") || !schemaKeys.contains("items")) {
86+
def msg = "${colors.red}The schema for '${samplesheetFile.toString()}' (${schemaFile.toString()}) is not valid. Please make sure that 'items' is the top level keyword and not 'properties'\n${colors.reset}\n"
87+
throw new SchemaValidationException(msg)
88+
}
89+
8390
if(!samplesheetFile.exists()) {
8491
def msg = "${colors.red}Samplesheet file ${samplesheetFile.toString()} does not exist\n${colors.reset}\n"
8592
throw new SchemaValidationException(msg)
8693
}
8794

8895
// Validate
8996
final validator = new JsonSchemaValidator(config)
90-
def JSONArray samplesheet = Utils.fileToJsonArray(samplesheetFile, schemaFile)
97+
def JSONArray samplesheet = Utils.fileToJson(samplesheetFile, schemaFile) as JSONArray
9198
def List<String> validationErrors = validator.validate(samplesheet, schemaFile.text)
9299
if (validationErrors) {
93100
def msg = "${colors.red}The following errors have been detected in ${samplesheetFile.toString()}:\n\n" + validationErrors.join('\n').trim() + "\n${colors.reset}\n"
@@ -96,8 +103,7 @@ class SamplesheetConverter {
96103
}
97104

98105
// Convert
99-
def LinkedHashMap schemaMap = new JsonSlurper().parseText(schemaFile.text) as LinkedHashMap
100-
def List samplesheetList = Utils.fileToList(samplesheetFile, schemaFile)
106+
def List samplesheetList = Utils.fileToObject(samplesheetFile, schemaFile) as List
101107

102108
this.rows = []
103109

‎plugins/nf-schema/src/main/nextflow/validation/Utils.groovy

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -52,13 +52,19 @@ public class Utils {
5252
}
5353

5454
// Converts a given file to a List
55-
public static List fileToList(Path file, Path schema) {
55+
public static Object fileToObject(Path file, Path schema) {
5656
def String fileType = Utils.getFileType(file)
5757
def String delimiter = fileType == "csv" ? "," : fileType == "tsv" ? "\t" : null
58+
def Map schemaMap = (Map) new JsonSlurper().parse( schema )
5859
def Map types = variableTypes(schema)
5960

60-
if (types.find{ it.value == "array" } as Boolean && fileType in ["csv", "tsv"]){
61-
def msg = "Using \"type\": \"array\" in schema with a \".$fileType\" samplesheet is not supported\n"
61+
if (schemaMap.type == "object" && fileType in ["csv", "tsv"]) {
62+
def msg = "CSV or TSV files are not supported. Use a JSON or YAML file instead of ${file.toString()}. (Expected a non-list data structure, which is not supported in CSV or TSV)"
63+
throw new SchemaValidationException(msg, [])
64+
}
65+
66+
if ((types.find{ it.value == "array" || it.value == "object" } as Boolean) && fileType in ["csv", "tsv"]){
67+
def msg = "Using \"type\": \"array\" or \"type\": \"object\" in schema with a \".$fileType\" samplesheet is not supported\n"
6268
log.error("ERROR: Validation of pipeline parameters failed!")
6369
throw new SchemaValidationException(msg, [])
6470
}
@@ -67,7 +73,7 @@ public class Utils {
6773
return new Yaml().load((file.text))
6874
}
6975
else if(fileType == "json"){
70-
return new JsonSlurper().parseText(file.text) as List
76+
return new JsonSlurper().parseText(file.text)
7177
}
7278
else {
7379
def Boolean header = getValueFromJson("#/items/properties", new JSONObject(schema.text)) ? true : false
@@ -82,13 +88,21 @@ public class Utils {
8288
}
8389

8490
// Converts a given file to a JSONArray
85-
public static JSONArray fileToJsonArray(Path file, Path schema) {
91+
public static Object fileToJson(Path file, Path schema) {
8692
// Remove all null values from JSON object
8793
// and convert the groovy object to a JSONArray
8894
def jsonGenerator = new JsonGenerator.Options()
8995
.excludeNulls()
9096
.build()
91-
return new JSONArray(jsonGenerator.toJson(fileToList(file, schema)))
97+
def Object obj = fileToObject(file, schema)
98+
if (obj instanceof List) {
99+
return new JSONArray(jsonGenerator.toJson(obj))
100+
} else if (obj instanceof Map) {
101+
return new JSONObject(jsonGenerator.toJson(obj))
102+
} else {
103+
def msg = "Could not determine if the file is a list or map of values"
104+
throw new SchemaValidationException(msg, [])
105+
}
92106
}
93107

94108
//
@@ -141,7 +155,7 @@ public class Utils {
141155
def Map parsed = (Map) slurper.parse( schema )
142156

143157
// Obtain the type of each variable in the schema
144-
def Map properties = (Map) parsed['items']['properties']
158+
def Map properties = (Map) parsed['items'] ? parsed['items']['properties'] : parsed["properties"]
145159
for (p in properties) {
146160
def String key = (String) p.key
147161
def Map property = properties[key] as Map

‎plugins/nf-schema/src/main/nextflow/validation/SchemaValidator.groovy renamed to ‎plugins/nf-schema/src/main/nextflow/validation/ValidationExtension.groovy

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ import org.yaml.snakeyaml.Yaml
3838

3939
@Slf4j
4040
@CompileStatic
41-
class SchemaValidator extends PluginExtensionPoint {
41+
class ValidationExtension extends PluginExtensionPoint {
4242

4343
final List<String> NF_OPTIONS = [
4444
// Options for base `nextflow` command
@@ -152,24 +152,6 @@ class SchemaValidator extends PluginExtensionPoint {
152152
// Help message logic
153153
def Map params = (Map)session.params ?: [:]
154154
config = new ValidationConfig(session?.config?.navigate('validation') as Map, params)
155-
def Boolean containsFullParameter = params.containsKey(config.help.fullParameter) && params[config.help.fullParameter]
156-
def Boolean containsShortParameter = params.containsKey(config.help.shortParameter) && params[config.help.shortParameter]
157-
if (config.help.enabled && (containsFullParameter || containsShortParameter)) {
158-
def String help = ""
159-
def HelpMessage helpMessage = new HelpMessage(config, session)
160-
help += helpMessage.getBeforeText()
161-
if (containsFullParameter) {
162-
log.debug("Printing out the full help message")
163-
help += helpMessage.getFullHelpMessage()
164-
} else if (containsShortParameter) {
165-
log.debug("Printing out the short help message")
166-
def paramValue = params.get(config.help.shortParameter)
167-
help += helpMessage.getShortHelpMessage(paramValue instanceof String ? paramValue : "")
168-
}
169-
help += helpMessage.getAfterText()
170-
log.info(help)
171-
System.exit(0)
172-
}
173155

174156
}
175157

‎plugins/nf-schema/src/main/nextflow/validation/ValidationObserver.groovy

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
package nextflow.validation
2+
3+
import groovy.util.logging.Slf4j
4+
5+
import nextflow.processor.TaskHandler
6+
import nextflow.trace.TraceObserver
7+
import nextflow.trace.TraceRecord
8+
import nextflow.Session
9+
10+
@Slf4j
11+
class ValidationObserver implements TraceObserver {
12+
13+
@Override
14+
void onFlowCreate(Session session) {
15+
// Help message logic
16+
def Map params = (Map)session.params ?: [:]
17+
def ValidationConfig config = new ValidationConfig(session?.config?.navigate('validation') as Map, params)
18+
def Boolean containsFullParameter = params.containsKey(config.help.fullParameter) && params[config.help.fullParameter]
19+
def Boolean containsShortParameter = params.containsKey(config.help.shortParameter) && params[config.help.shortParameter]
20+
if (config.help.enabled && (containsFullParameter || containsShortParameter)) {
21+
def String help = ""
22+
def HelpMessage helpMessage = new HelpMessage(config, session)
23+
help += helpMessage.getBeforeText()
24+
if (containsFullParameter) {
25+
log.debug("Printing out the full help message")
26+
help += helpMessage.getFullHelpMessage()
27+
} else if (containsShortParameter) {
28+
log.debug("Printing out the short help message")
29+
def paramValue = params.get(config.help.shortParameter)
30+
help += helpMessage.getShortHelpMessage(paramValue instanceof String ? paramValue : "")
31+
}
32+
help += helpMessage.getAfterText()
33+
log.info(help)
34+
System.exit(0)
35+
}
36+
}
37+
}

‎plugins/nf-schema/src/main/nextflow/validation/ValidationObserverFactory.groovy

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
package nextflow.validation
2+
3+
import nextflow.Session
4+
import nextflow.trace.TraceObserver
5+
import nextflow.trace.TraceObserverFactory
6+
7+
class ValidationObserverFactory implements TraceObserverFactory {
8+
9+
@Override
10+
Collection<TraceObserver> create(Session session) {
11+
// Only enable the trace observer when a help message needs to be printed
12+
final enabled = session.config.navigate('validation.help.enabled')
13+
return enabled ? [ new ValidationObserver() ] : []
14+
}
15+
}

‎plugins/nf-schema/src/resources/META-INF/MANIFEST.MF

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
Manifest-Version: 1.0
22
Plugin-Id: nf-schema
3-
Plugin-Version: 2.2.1
3+
Plugin-Version: 2.3.0
44
Plugin-Class: nextflow.validation.ValidationPlugin
55
Plugin-Provider: nextflow
66
Plugin-Requires: >=23.10.0

‎plugins/nf-schema/src/resources/META-INF/extensions.idx

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
1-
nextflow.validation.SchemaValidator
1+
nextflow.validation.ValidationExtension
2+
nextflow.validation.ValidationObserverFactory

‎plugins/nf-schema/src/test/nextflow/validation/ValidateParametersTest.groovy

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1224,4 +1224,102 @@ class ValidateParametersTest extends Dsl2Spec{
12241224
!stdout
12251225
}
12261226

1227+
def 'should validate a map file - yaml' () {
1228+
given:
1229+
def schema = Path.of('src/testResources/nextflow_schema_with_map_file.json').toAbsolutePath().toString()
1230+
def SCRIPT = """
1231+
params.map_file = 'src/testResources/map_file.yaml'
1232+
include { validateParameters } from 'plugin/nf-schema'
1233+
1234+
validateParameters(parameters_schema: '$schema')
1235+
"""
1236+
1237+
when:
1238+
def config = [:]
1239+
def result = new MockScriptRunner(config).setScript(SCRIPT).execute()
1240+
def stdout = capture
1241+
.toString()
1242+
.readLines()
1243+
.findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
1244+
1245+
then:
1246+
noExceptionThrown()
1247+
!stdout
1248+
}
1249+
1250+
def 'should validate a map file - json' () {
1251+
given:
1252+
def schema = Path.of('src/testResources/nextflow_schema_with_map_file.json').toAbsolutePath().toString()
1253+
def SCRIPT = """
1254+
params.map_file = 'src/testResources/map_file.json'
1255+
include { validateParameters } from 'plugin/nf-schema'
1256+
1257+
validateParameters(parameters_schema: '$schema')
1258+
"""
1259+
1260+
when:
1261+
def config = [:]
1262+
def result = new MockScriptRunner(config).setScript(SCRIPT).execute()
1263+
def stdout = capture
1264+
.toString()
1265+
.readLines()
1266+
.findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
1267+
1268+
then:
1269+
noExceptionThrown()
1270+
!stdout
1271+
}
1272+
1273+
def 'should give an error when a map file is wrong - yaml' () {
1274+
given:
1275+
def schema = Path.of('src/testResources/nextflow_schema_with_map_file.json').toAbsolutePath().toString()
1276+
def SCRIPT = """
1277+
params.map_file = 'src/testResources/map_file_wrong.yaml'
1278+
include { validateParameters } from 'plugin/nf-schema'
1279+
1280+
validateParameters(parameters_schema: '$schema')
1281+
"""
1282+
1283+
when:
1284+
def config = [:]
1285+
def result = new MockScriptRunner(config).setScript(SCRIPT).execute()
1286+
def stdout = capture
1287+
.toString()
1288+
.readLines()
1289+
.findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
1290+
1291+
1292+
then:
1293+
def error = thrown(SchemaValidationException)
1294+
error.message.contains("* --map_file (src/testResources/map_file_wrong.yaml): Validation of file failed:")
1295+
error.message.contains(" * --this.is.deep (hello): Value is [string] but should be [integer]")
1296+
!stdout
1297+
}
1298+
1299+
def 'should give an error when a map file is wrong - json' () {
1300+
given:
1301+
def schema = Path.of('src/testResources/nextflow_schema_with_map_file.json').toAbsolutePath().toString()
1302+
def SCRIPT = """
1303+
params.map_file = 'src/testResources/map_file_wrong.json'
1304+
include { validateParameters } from 'plugin/nf-schema'
1305+
1306+
validateParameters(parameters_schema: '$schema')
1307+
"""
1308+
1309+
when:
1310+
def config = [:]
1311+
def result = new MockScriptRunner(config).setScript(SCRIPT).execute()
1312+
def stdout = capture
1313+
.toString()
1314+
.readLines()
1315+
.findResults {it.contains('WARN nextflow.validation.SchemaValidator') || it.startsWith('* --') ? it : null }
1316+
1317+
1318+
then:
1319+
def error = thrown(SchemaValidationException)
1320+
error.message.contains("* --map_file (src/testResources/map_file_wrong.json): Validation of file failed:")
1321+
error.message.contains(" * --this.is.deep (hello): Value is [string] but should be [integer]")
1322+
!stdout
1323+
}
1324+
12271325
}

‎plugins/nf-schema/src/testResources/map_file.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"test": "hello",
3+
"this": {
4+
"is": {
5+
"deep": 20
6+
}
7+
}
8+
}

‎plugins/nf-schema/src/testResources/map_file.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
test: hello
2+
this:
3+
is:
4+
deep: 20

‎plugins/nf-schema/src/testResources/map_file_wrong.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"test": "hello",
3+
"this": {
4+
"is": {
5+
"deep": "hello"
6+
}
7+
}
8+
}

‎plugins/nf-schema/src/testResources/map_file_wrong.yaml

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
test: hello
2+
this:
3+
is:
4+
deep: hello

‎plugins/nf-schema/src/testResources/nextflow_schema_with_map_file.json

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"$schema": "https://json-schema.org/draft/2020-12/schema",
3+
"$id": "https://raw.githubusercontent.com/nf-core/testpipeline/master/nextflow_schema.json",
4+
"title": "nf-core/testpipeline pipeline parameters",
5+
"description": "this is a test",
6+
"type": "object",
7+
"properties": {
8+
"map_file": {
9+
"type": "string",
10+
"format": "file-path",
11+
"schema": "src/testResources/schema_map_file.json"
12+
}
13+
}
14+
}

0 commit comments

Comments
 (0)