Skip to content
This repository was archived by the owner on Aug 20, 2024. It is now read-only.

Commit 18cdbb2

Browse files
ysb33rysb33r
committed
Merge pull request #70 from ysb33r/development
Merging Beta-7 changes
2 parents b613310 + e595b33 commit 18cdbb2

File tree

10 files changed

+147
-47
lines changed

10 files changed

+147
-47
lines changed

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ buildscript {
2626

2727
allprojects {
2828
ext {
29-
versionModifier = 'beta6'
29+
versionModifier = 'beta7'
3030
versionNumber = '1.0'
3131
modulesWithGroovyDoc = [
3232
'dsl',

‎docs/src/docs/asciidoc/gradle-plugin.adoc‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ in order to determine whether the task is out of date
8787
* The source file is newer than the destination (folder timestamps are not checked)
8888

8989
NOTE: In beta-3, beta-4 input-output caching is not working as yet. https://github.com/ysb33r/groovy-vfs/issues/49[#49]. This
90-
was rectified in beta-5.
90+
was rectified in beta-5. IN was updated again in beta-7 https://github.com/ysb33r/groovy-vfs/issues/64[#64].
9191

9292
=== Copy optimisations
9393

‎dsl/RELEASE.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,9 +12,12 @@
1212
+ [Add option to mkdir to not automatically create intermediate paths](https://github.com/ysb33r/groovy-vfs/issues/18)
1313
+ [Send arbitrary content to a remote file](https://github.com/ysb33r/groovy-vfs/issues/39)
1414
+ [uri keyword](https://github.com/ysb33r/groovy-vfs/issues/38)
15+
+ [Up to date check related to GET_LAST_MODIFIED](https://github.com/ysb33r/groovy-vfs/issues/64)
1516
+ Support for `mkdir`
1617
+ `defaultProvider` will be ignored when constructing a `VFS` object. Use `provider` configuration closure instead.
1718
+ Pass `antPattern` to `filter` to use ANT-style patterns when copying files
19+
+ Added boolean property `closeFileSystem` that can be passed to `ls` method.
20+
+ Fixed issue when resolved URI are passed to some methods with additional non-VFS options
1821

1922
## 0.5
2023
+ Removed standalone use of InputStream when applying `cat`.

‎dsl/src/main/groovy/org/ysb33r/groovy/dsl/vfs/VFS.groovy‎

Lines changed: 24 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,8 @@ class VFS {
156156
*/
157157
VFS( Map properties=[:], Closure pluginLoader = null ) {
158158
fsMgr = new StandardFileSystemManager()
159-
159+
160+
160161
Log vfslog = properties.containsKey('logger') ? (properties['logger'] as Log): new NoOpLog()
161162

162163
if(properties.containsKey('defaultProvider')) {
@@ -231,9 +232,10 @@ class VFS {
231232
*
232233
* @param properties
233234
* @li filter A regex, closure of VFS FileSelector by which to select objects
234-
* @li recursive If set to true will traverse down any subfolders. Ignored if filter is a FileSelector
235+
* @li recursive If set to true will traverse down any subfolders. Ignored if filter is a FileSelector
236+
* @li closeFilesystem Closes underlying filesystem when returning. Default is not to.
235237
*
236-
* @param uri URI pointing to area for which file listign is to be retrieved
238+
* @param uri URI pointing to area for which file listing is to be retrieved
237239
* @param c Closure that will be a single parameter (VFS FileObject)
238240
*
239241
* @return If a closure is specified returns the results of passing each file to the
@@ -259,7 +261,8 @@ class VFS {
259261
def ls ( Map properties=[:],uri,Closure c ) {
260262
assert properties != null
261263
def children
262-
def ruri=resolveURI(properties,uri)
264+
FileObject ruri=resolveURI(properties,uri)
265+
AbstractFileSystem afs= properties.closeFilesystem ? ruri.fileSystem as AbstractFileSystem : null
263266
boolean recurse = properties.containsKey('recursive') ? (properties['recursive'] as boolean): false
264267

265268
if( properties.containsKey('filter') ) {
@@ -297,13 +300,17 @@ class VFS {
297300
else {
298301
children= ruri.children
299302
}
300-
301-
if(c) {
302-
Closure newc=c.clone()
303-
newc.delegate=this
304-
return children.collect { newc.call(it) }
305-
} else {
306-
return children
303+
304+
try {
305+
if(c) {
306+
Closure newc=c.clone()
307+
newc.delegate=this
308+
return children.collect { newc.call(it) }
309+
} else {
310+
return children
311+
}
312+
} finally {
313+
afs?.closeCommunicationLink()
307314
}
308315
}
309316

@@ -732,7 +739,12 @@ class VFS {
732739
@CompileDynamic
733740
FileObject resolveURI (Map properties=[:],uri) {
734741
if (uri instanceof FileObject) {
735-
properties.size() ? Util.resolveURI(properties,fsMgr,uri.fileSystem.fileSystemOptions,uri.name.getURI()) : uri
742+
if( properties.size() ) {
743+
Map vfsProperties = Util.selectVfsOptions(properties)
744+
vfsProperties.size() ? Util.resolveURI(vfsProperties,fsMgr,uri.fileSystem.fileSystemOptions,uri.name.getURI()) : uri
745+
} else {
746+
return uri
747+
}
736748
} else {
737749
Util.resolveURI(properties,fsMgr,defaultFSOptions,uri)
738750
}

‎dsl/src/main/groovy/org/ysb33r/groovy/dsl/vfs/impl/Util.groovy‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,13 @@ import org.apache.commons.vfs2.provider.TemporaryFileStore
4141
import org.apache.commons.vfs2.impl.DefaultFileReplicator
4242
import org.ysb33r.groovy.dsl.vfs.URIException
4343

44+
import java.util.regex.Pattern
45+
4446
@CompileStatic
4547
class Util {
4648

49+
static final Pattern OPTION_REGEX = ~/^(?i:vfs\.)(\p{Alpha}\p{Alnum}+)\.(\p{Alpha}\w+)$/
50+
4751
/** Resolves a file from a URI.
4852
* @param properties Map of 'vfs.SCHEME.OPTION' properties. ANy options passed in here will override options in defaultFSOptions.
4953
* @param fsMgr Apache VFS FileSystemManager instance
@@ -58,6 +62,23 @@ class Util {
5862
fsMgr.resolveFile(u.toString(), properties ? this.buildOptions(properties,fsMgr,fo) : fo )
5963
}
6064

65+
/** Returns a map only containing vfs options
66+
*
67+
* @param options Map with any kinds of options including options starting with vfs.
68+
* @return Filtered Map
69+
*/
70+
@CompileDynamic
71+
static Map<String,Object> selectVfsOptions(final Map<String,Object> options) {
72+
Map<String,Object> filteredMap = [:]
73+
options.each { k,v ->
74+
def m = k =~ OPTION_REGEX
75+
if (m.matches()) {
76+
filteredMap[k] = v
77+
}
78+
}
79+
filteredMap
80+
}
81+
6182
/** Traverses a map extracting keys in the form of 'vfs.SCHEME.FSOPTION'.
6283
* Keys not of this form or not supported by the current file system
6384
* manager will be ignored.
@@ -72,7 +93,7 @@ class Util {
7293
static def buildOptions (Map options,FileSystemManager fsMgr, FileSystemOptions baseFSOpt=null) {
7394
def fsOpt = baseFSOpt ? baseFSOpt.clone() : new FileSystemOptions()
7495
options.each { k,v ->
75-
def m = k =~ /^(?i:vfs\.)(\p{Alpha}\p{Alnum}+)\.(\p{Alpha}\w+)$/
96+
def m = k =~ OPTION_REGEX
7697
if (m.matches()) {
7798
def scheme = m[0][1]
7899
def opt = m[0][2]

‎dsl/src/test/groovy/org/ysb33r/groovy/dsl/vfs/FtpSpec.groovy‎

Lines changed: 37 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,21 +11,10 @@
1111
*
1212
* ============================================================================
1313
*/
14-
// ============================================================================
15-
// (C) Copyright Schalk W. Cronje 2013
16-
//
17-
// This software is licensed under the Apache License 2.0
18-
// See http://www.apache.org/licenses/LICENSE-2.0 for license details
19-
//
20-
// Unless required by applicable law or agreed to in writing, software distributed under the License is
21-
// distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22-
// See the License for the specific language governing permissions and limitations under the License.
23-
//
24-
// ============================================================================
25-
2614
package org.ysb33r.groovy.dsl.vfs
2715

2816
import org.apache.commons.vfs2.FileObject
17+
import org.ysb33r.groovy.dsl.vfs.impl.Util
2918
import spock.lang.*
3019
import org.ysb33r.groovy.dsl.vfs.services.*
3120
import org.ysb33r.groovy.dsl.vfs.helpers.*
@@ -70,4 +59,40 @@ class FtpSpec extends SchemaSpec {
7059

7160
}
7261

62+
def "Resolving a resolved URI with encoded password must resolve correctly"() {
63+
given: "A previously resolved URI"
64+
FileObject firstResolve = vfs.resolveURI("ftp://guest:{6D9D9D4A32C1C3F9B0FCDC0162476BAA}@localhost:${server.PORT}/test-subdir?vfs.ftp.passiveMode=1")
65+
66+
when:
67+
def secondResolve = vfs.resolveURI(firstResolve)
68+
69+
then:
70+
secondResolve != null
71+
72+
when:
73+
def thirdResolve = Util.resolveURI(
74+
[ filter : ~/.+/ ],
75+
firstResolve.fileSystem.fileSystemManager,
76+
firstResolve.fileSystem.fileSystemOptions,
77+
firstResolve.name.getURI()
78+
)
79+
80+
then:
81+
thirdResolve != null
82+
}
83+
84+
def "Listing a directory with a resolved URI and additional non-VFS options"() {
85+
given: "A previously resolved URI"
86+
def resolvedUri = vfs.resolveURI("ftp://guest:{6D9D9D4A32C1C3F9B0FCDC0162476BAA}@127.0.0.1:${server.PORT}/test-subdir?vfs.ftp.passiveMode=1&vfs.ftp.fooParam=1")
87+
List<String> names = []
88+
89+
when:
90+
vfs.ls resolvedUri, filter : ~/.+/, { FileObject fo ->
91+
names+= fo.name.baseName
92+
}
93+
94+
then:
95+
names.size()
96+
}
97+
7398
}

‎gradle-plugin/src/main/groovy/org/ysb33r/gradle/vfs/internal/UpToDateCheck.groovy‎

Lines changed: 42 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import groovy.transform.CompileStatic
1818
import org.apache.commons.vfs2.Capability
1919
import org.apache.commons.vfs2.FileName
2020
import org.apache.commons.vfs2.FileObject
21+
import org.apache.commons.vfs2.FileSystem
2122
import org.apache.commons.vfs2.FileSystemException
2223
import org.gradle.api.logging.Logger
2324
import org.ysb33r.gradle.vfs.VfsCopySpec
@@ -34,6 +35,7 @@ import org.ysb33r.groovy.dsl.vfs.impl.Util
3435
@CompileStatic
3536
class UpToDateCheck {
3637

38+
3739
/** Iterates over all of the file hierarchy within in {@link VfsURICollection} and checks whether the equivalent file
3840
* would exist on a target. If the source and remote schemes support GET_MODIFIED_DATE, then those dates would also be
3941
* used as part of the up-to-date decision
@@ -42,14 +44,21 @@ class UpToDateCheck {
4244
* @param vfs
4345
* @param sources
4446
* @param destRoot
47+
* @param noSourceModifiedDateCheck Don't check the source modification dates
4548
* @return {@code false} as soon as the first remote target does not exist or is older than the source.
4649
*/
47-
static boolean forUriCollection( Logger logger, VFS vfs, VfsURICollection sources, def destRoot) {
50+
static boolean forUriCollection(
51+
Logger logger,
52+
VFS vfs,
53+
VfsURICollection sources,
54+
def destRoot,
55+
boolean noSourceModifiedDateCheck
56+
) {
4857

4958
final FileObject destFileObject = vfs.resolveURI(destRoot)
50-
final boolean destHasModifiedDate = destFileObject.fileSystem.hasCapability(Capability.GET_LAST_MODIFIED)
59+
final boolean destHasModifiedDate = filesystemHasLastModifiedDate(destFileObject)
5160

52-
// Using exceptions for flow control. I don;t really like it,
61+
// Using exceptions for flow control. I don't really like it,
5362
// but currently it is the only way to go as vfs.ls does not implement an Iterator interface
5463
try {
5564
sources.each { vfsURI ->
@@ -58,8 +67,7 @@ class UpToDateCheck {
5867
FileName srcRoot = srcFileObject.name
5968
def options = [:]
6069

61-
boolean checkModifiedDate = destHasModifiedDate &&
62-
srcFileObject.fileSystem.hasCapability(Capability.GET_LAST_MODIFIED)
70+
boolean checkModifiedDate = !noSourceModifiedDateCheck && destHasModifiedDate && filesystemHasLastModifiedDate(srcFileObject)
6371

6472
if(vfs.fsCanListFolderContent(src.uri)) {
6573
if (src.praxis.filter) {
@@ -83,31 +91,55 @@ class UpToDateCheck {
8391
return true
8492
}
8593

86-
/** Checks whether the source is newer than the destination. Only dayes on files are checked, folders are not.
94+
/** Checks whether the source is newer than the destination. Only dates on files are checked, folders are not.
8795
*
8896
* @param logger
8997
* @param vfs
9098
* @param rootSpec
9199
* @param destRoot
100+
* @param noSourceModifiedDateCheck Don't check the source modification dates
92101
* @return {@code true} is destination is considered older than source
93102
*/
94-
static boolean forCopySpec( Logger logger, VFS vfs, VfsCopySpec rootSpec, VfsURI destRoot ) {
103+
static boolean forCopySpec( Logger logger, VFS vfs, VfsCopySpec rootSpec, VfsURI destRoot, boolean noSourceModifiedDateCheck ) {
95104
VfsURI dest = destRoot.resolve()
96105
if(!vfs.exists(destRoot.uri)) {
97106
logger.debug "Target is out of date: ${friendlyURI(vfs,dest)} does not exist."
98107
return false
99108
}
100109

101-
if(!forUriCollection( logger, vfs,rootSpec.uriCollection , destRoot.uri)) {
110+
if(!forUriCollection( logger, vfs,rootSpec.uriCollection , destRoot.uri,noSourceModifiedDateCheck)) {
111+
logger.debug "Sources for rootSpec has changed"
102112
return false
103113
}
104114

105115
for( Object ch : rootSpec.children()) {
106116
VfsCopySpec child = ch as VfsCopySpec
107117
VfsURI childDest = ResolvedURI.create([:], vfs, Util.addRelativePath(dest.uri as FileObject, child.relativePath))
108-
if(!forUriCollection(logger,vfs,child.uriCollection,childDest.uri)) {
118+
if(!forUriCollection(logger,vfs,child.uriCollection,childDest.uri,noSourceModifiedDateCheck)) {
119+
logger.debug "Sources targeting child target ${friendlyURI(vfs,childDest)} has changed"
120+
return false
121+
}
122+
}
123+
124+
logger.debug "Target ${friendlyURI(vfs,dest)} is up to date"
125+
return true
126+
}
127+
128+
/** Checks whether a single-layer or multi-layered filesystem can chekc the modified date throughout
129+
*
130+
* @param fo VFS2 FileObject (must not be null)
131+
* @return {@code true} is all layers can supprot {@code GET_LAST_MODIFIED_DATE}
132+
*/
133+
private static boolean filesystemHasLastModifiedDate(FileObject fo) {
134+
assert fo != null
135+
while(fo != null) {
136+
FileSystem fs = fo.fileSystem
137+
138+
if (!fs.hasCapability(Capability.GET_LAST_MODIFIED)) {
109139
return false
110140
}
141+
142+
fo = fs.parentLayer
111143
}
112144

113145
return true
@@ -121,7 +153,7 @@ class UpToDateCheck {
121153
* @param dest Destination {@code FileObject}
122154
* @param checkModifiedDate Whether modification dates should be compared
123155
*
124-
* @throw OutOfDateException is destination is deemded out of date
156+
* @throw OutOfDateException if destination is deemed out of date
125157
*
126158
*/
127159
private static void throwIfOutOfDate( VFS vfs, Object src, FileObject dest, boolean checkModifiedDate) {

‎gradle-plugin/src/main/groovy/org/ysb33r/gradle/vfs/internal/VfsBaseTask.groovy‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -92,27 +92,26 @@ abstract class VfsBaseTask extends DefaultTask {
9292
this.options+= opts
9393
}
9494

95-
@Input
96-
Map<String,Object> getPraxis() {
95+
VFS getVfs() {
96+
this.vfs
97+
}
98+
99+
protected Map<String,Object> getPraxis() {
97100
this.praxis ?: defaultPraxis()
98101
}
99102

100-
Map<String,Object> praxis( Map<String,Object> opts) {
103+
protected Map<String,Object> praxis( Map<String,Object> opts) {
101104
if(this.praxis == null) {
102105
this.praxis = opts
103106
} else {
104107
this.praxis += opts
105108
}
106109
}
107110

108-
Map<String,Object> setPraxis( Map<String,Object> opts) {
111+
protected Map<String,Object> setPraxis( Map<String,Object> opts) {
109112
this.praxis = opts
110113
}
111114

112-
VFS getVfs() {
113-
this.vfs
114-
}
115-
116115
protected VfsURI stage(Object uri) {
117116
UriUtils.uriWithOptions(getOptions(),vfs,uri)
118117
}

‎gradle-plugin/src/main/groovy/org/ysb33r/gradle/vfs/tasks/VfsCopy.groovy‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,14 @@ import org.ysb33r.groovy.dsl.vfs.VFS
4141
@CompileStatic
4242
class VfsCopy extends VfsBaseTask {
4343

44+
/** Certain remote systems do not report modified dates correctly, resulting in unnecessary download.
45+
* If this is the case date checks can be turned off for as task instance. This will result in files only being
46+
* downloaded if the target does not exists or {@code --rerun-tasks} is passed.
47+
*/
48+
@Input
49+
boolean noSourceModifiedDateCheck = false
50+
51+
4452
/** Checks the state of remote objects and decides whether the object can be up to date.
4553
* Up to date can be considered for the following considerations
4654
* <ul>
@@ -55,7 +63,7 @@ class VfsCopy extends VfsBaseTask {
5563
boolean isUpToDate() {
5664
Map<String,Object> opts = getOptions() + getPraxis()
5765
copySpec.apply( [getOptionMap : { -> opts } ] as VfsOptions )
58-
return UpToDateCheck.forCopySpec(logger,super.vfs,copySpec,getDestination())
66+
return UpToDateCheck.forCopySpec(logger,super.vfs,copySpec,getDestination(),noSourceModifiedDateCheck)
5967
}
6068

6169
/** Returns a default set of VFS action options (praxis) in case no task-wide set if defined for all URIs

‎website‎

Submodule website updated 130 files

0 commit comments

Comments
 (0)