Merge pull request #381 from chrisfilo/enh/bidsignore

Implementation of .bidsignore

Author: chrisgorgo

README.md

@@ -44,6 +44,11 @@ If you would like to test individual files you can use the file specific checks
 
 Additionally you can reformat stored errors against a new config using `validate.reformat()`
 
+#### .bidsignore
+Optionally one can include a `.bidsignore` file in the root of the dataset. This file lists patterns (compatible
+with the [.gitignore syntax](https://git-scm.com/docs/gitignore)) defining files that should be ignored by the
+validator. This option is useful when the validated dataset includes file types not yet supported by BIDS specification.
+
 #### Configuration
 
 You can configure the severity of errors by passing a json configuration file with a --c or --config flag to the command line interface or by defining a config object on the options object passed during javascript usage.

package.json

@@ -33,7 +33,8 @@
     "pako": "^1.0.4",
     "path": "^0.12.7",
     "pluralize": "^3.1.0",
-    "yargs": "^6.6.0"
+    "yargs": "^6.6.0",
+    "ignore": "^3.3.7"
   },
   "devDependencies": {
     "adm-zip": "",

tests/data/valid_headers/.bidsignore

@@ -0,0 +1 @@
+*_not_bids.txt

tests/data/valid_headers/sub-01/func/extra_file_that_is_not_bids.txt

@@ -2,7 +2,8 @@
 
 var nifti = require('nifti-js');
 var Issue = require('./issues').Issue;
-var type =  require('./type');
+var ignore = require('ignore');
+var path = require('path');
 
 /**
  * If the current environment is server side
@@ -65,7 +66,61 @@ function readFile (file, callback) {
     }
 }
 
+function getBIDSIgnoreFileObjNode(dir) {
+    var bidsIgnoreFileObj = null;
+    var path = dir + "/.bidsignore";
+    if (fs.existsSync(path)) {
+        bidsIgnoreFileObj = {path: path};
+    }
+    return bidsIgnoreFileObj;
+}
+
+function getBIDSIgnoreFileObjBrowser(dir) {
+    var bidsIgnoreFileObj = null;
+    for (var i = 0; i < dir.length; i++) {
+        var fileObj = dir[i];
+        var relativePath = harmonizeRelativePath(fileObj.webkitRelativePath);
+        if (relativePath === "/.bidsignore") {
+            bidsIgnoreFileObj = fileObj;
+            break;
+        }
+    }
+    return bidsIgnoreFileObj;
+}
+
+/**
+ * Get File object corresponding to the .bidsignore file
+ * @param dir
+ * @returns File object or null if not found
+ */
+function getBIDSIgnoreFileObj(dir) {
+    var bidsIgnoreFileObj = null;
+    if (fs) {
+        bidsIgnoreFileObj = getBIDSIgnoreFileObjNode(dir);
+    } else {
+        bidsIgnoreFileObj = getBIDSIgnoreFileObjBrowser(dir);
+    }
+    return bidsIgnoreFileObj;
+}
 
+function getBIDSIgnore(dir, callback) {
+    var ig = ignore()
+        .add('/derivatives')
+        .add('/sourcedata')
+        .add('/code')
+        .add('.*');
+
+    var bidsIgnoreFileObj = getBIDSIgnoreFileObj(dir);
+    if (bidsIgnoreFileObj) {
+        readFile(bidsIgnoreFileObj, function (issue, content) {
+            ig = ig.add(content);
+            callback(ig);
+        });
+    } else {
+        callback(ig);
+    }
+    return ig;
+}
 
 /**
  * Read Directory
@@ -80,16 +135,20 @@ function readFile (file, callback) {
 function readDir (dir, callback) {
     var filesObj = {};
     var filesList = [];
-    if (fs) {
-        filesList = preprocessNode(dir);
-    } else {
-        filesList = preprocessBrowser(dir);
-    }
-    // converting array to object
-    for (var j = 0; j < filesList.length; j++) {
-        filesObj[j] = filesList[j];
+
+    function callbackWrapper(ig) {
+        if (fs) {
+            filesList = preprocessNode(dir, ig);
+        } else {
+            filesList = preprocessBrowser(dir, ig);
+        }
+        // converting array to object
+        for (var j = 0; j < filesList.length; j++) {
+            filesObj[j] = filesList[j];
+        }
+        callback(filesObj);
     }
-    callback(filesObj);
+    getBIDSIgnore(dir, callbackWrapper);
 }
 
 /**
@@ -98,12 +157,12 @@ function readDir (dir, callback) {
  * 1. Filters out ignored files and folder.
  * 2. Adds 'relativePath' field of each file object.
  */
-function preprocessBrowser(filesObj) {
+function preprocessBrowser(filesObj, ig) {
     var filesList = [];
     for (var i = 0; i < filesObj.length; i++) {
         var fileObj = filesObj[i];
         fileObj.relativePath = harmonizeRelativePath(fileObj.webkitRelativePath);
-        if (type.isIgnoredPath(fileObj.relativePath)) {
+        if (ig.ignores(path.relative('/', fileObj.relativePath))) {
             continue;
         }
         filesList.push(fileObj);
@@ -118,23 +177,23 @@ function preprocessBrowser(filesObj) {
  * 2. Filters out ignored files and folder.
  * 3. Harmonizes the 'relativePath' field
  */
-function preprocessNode(dir) {
+function preprocessNode(dir, ig) {
     var str = dir.substr(dir.lastIndexOf('/') + 1) + '$';
     var rootpath = dir.replace(new RegExp(str), '');
-    return getFiles(dir, [], rootpath);
+    return getFiles(dir, [], rootpath, ig);
 }
 
 /**
  * Recursive helper function for 'preprocessNode'
  */
-function getFiles(dir, files_, rootpath){
+function getFiles(dir, files_, rootpath, ig){
     files_ = files_ || [];
     var files = fs.readdirSync(dir);
     for (var i = 0; i < files.length; i++) {
         var fullPath = dir + '/' + files[i];
         var relativePath = fullPath.replace(rootpath, '');
         relativePath = harmonizeRelativePath(relativePath);
-        if (type.isIgnoredPath(relativePath)){
+        if (ig.ignores(path.relative('/', relativePath))) {
             continue;
         }
         var fileName = files[i];
@@ -146,7 +205,7 @@ function getFiles(dir, files_, rootpath){
         };
 
         if (fs.lstatSync(fullPath).isDirectory()) {
-            getFiles(fullPath, files_, rootpath);
+            getFiles(fullPath, files_, rootpath, ig);
         } else {
             files_.push(fileObj);
         }

utils/files.js

@@ -9,7 +9,12 @@ module.exports = {
     1: {
         key: 'NOT_INCLUDED',
         severity: 'error',
-        reason: 'Files with such naming scheme are not part of BIDS specification. This error is most commonly caused by typos in file names that make them not BIDS compatible. Please consult the specification and make sure your files are named correctly. If this is not a file naming issue (for example when including files not yet covered by the BIDS specification) you can ignore this warning. Please note that derived (processed) data should be placed in /derivatives folder and source data (such as DICOMS or behavioural logs in proprietary formats) should be placed in the /sourcedata folder.'
+        reason: 'Files with such naming scheme are not part of BIDS specification. This error is most commonly ' +
+        'caused by typos in file names that make them not BIDS compatible. Please consult the specification and ' +
+        'make sure your files are named correctly. If this is not a file naming issue (for example when including ' +
+        'files not yet covered by the BIDS specification) you should include a ".bidsignore" file in your dataset. Please ' +
+        'note that derived (processed) data should be placed in /derivatives folder and source data (such as DICOMS ' +
+        'or behavioural logs in proprietary formats) should be placed in the /sourcedata folder.'
     },
     2: {
         key: 'REPETITION_TIME_GREATER_THAN',

utils/issues/list.js

@@ -199,11 +199,6 @@ module.exports = {
         return !isNaN(parseFloat(n)) && isFinite(n);
     },
 
-    isIgnoredPath: function (path) {
-        var ignoredDirsRe = new RegExp('^\\/(derivatives|sourcedata|code).*$');
-        var ignoreHiddenRe = new RegExp('^.*\\/[\\.].+$');
-        return conditionalMatch(ignoredDirsRe, path) || conditionalMatch(ignoreHiddenRe, path);
-    },
 
     /**
      * Get Path Values