@@ -48,6 +48,69 @@ def log_print(message):
4848# add script root to support import of URL and git maps
4949sys .path .append (script_root )
5050from dependency_map import git_mapping
51+ from dependency_map import url_mapping_win
52+ from dependency_map import url_mapping_linux
53+
54+ # Download a zip or tgz file from the specified URL and unzip into the directory defined by destination.
55+ # The destination directory will be created if it doesn't exist
56+ # if the 'update' parameter is true then the existing file and output directory will be deleted and re-created
57+ # TODO - this function needs to handle errors gracefully when URL is incorrect or inaccessible
58+ def download_url_dependencies (url_mapping , update , retry_count = 10 ):
59+ for url in url_mapping :
60+ # convert targetPath to OS specific format
61+ tmp_path = os .path .join (script_root , url_mapping [url ])
62+
63+ # clean up path, collapsing any ../ and converting / to \ for Windows
64+ target_path = os .path .normpath (tmp_path )
65+
66+ # TODO if update is defined - delete file and directory if they exist
67+
68+ # make target directory if it doesn't exist
69+ if not os .path .isdir (target_path ):
70+ os .makedirs (target_path )
71+
72+ # generate the target zip file name from the source URL filename and the target path
73+ # note - this rule currently handles URLs that contain # and ? characters at the end
74+ # those currently used by Jenkins don't have this style
75+ zip_file_name = url .split ('/' )[- 1 ].split ('#' )[0 ].split ('?' )[0 ]
76+ zip_path = os .path .join (target_path , zip_file_name )
77+
78+ # Build folder name for the zip_path.
79+ zip_file_path_list = zip_file_name .split ("." )
80+ zip_file_path = os .path .join (target_path , zip_file_path_list [0 ])
81+
82+ if (os .path .isfile (zip_path )):
83+ # If the archive file exists, print message and continue
84+ log_print ("URL Dependency %s found and not updated" % zip_path )
85+ elif (os .path .isdir (zip_file_path )):
86+ # If a folder of the same name as the archive exists, print message and continue
87+ log_print ("URL Dependency %s found and not updated" % zip_file_path )
88+ else :
89+ # File doesn't exist - download and unpack it
90+ log_print ("Downloading " + url + " into " + zip_path )
91+ try :
92+ urllib .urlretrieve (url , zip_path )
93+ except urllib .ContentTooShortError :
94+ os .remove (zip_path )
95+ if retry_count > 0 :
96+ log_print ("URL content too short. Retrying. Retries remaining: %d" % retry_count )
97+ download_url_dependencies (url_mapping , update , retry_count - 1 )
98+ return
99+
100+ # Unpack the downloaded file into the target directory
101+ extension = os .path .splitext (zip_path )[1 ]
102+ if extension == ".zip" :
103+ # if file extension is .zip then unzip it
104+ log_print ("Extracting in " + target_path )
105+ zipfile .ZipFile (zip_path ).extractall (target_path )
106+ # delete downloaded zip file
107+ os .remove (zip_path )
108+ elif extension == ".tgz" or extension == ".gz" :
109+ # if file extension is .tgz then untar it
110+ log_print ("Extracting in " + target_path )
111+ tarfile .open (zip_path ).extractall (target_path )
112+ # delete downloaded tgz file
113+ os .remove (zip_path )
51114
52115# Clone or update a git repo
53116def update_git_dependencies (git_mapping , update ):
@@ -115,6 +178,10 @@ def do_fetch_dependencies(update):
115178
116179 # Update all git dependencies
117180 if update_git_dependencies (git_mapping , update ):
181+ if sys .platform == "win32" :
182+ download_url_dependencies (url_mapping_win , update )
183+ elif sys .platform .startswith ('linux' ) == True :
184+ download_url_dependencies (url_mapping_linux , update )
118185 return True
119186 else :
120187 return False
0 commit comments