__author__ = 'xgiovio' import authentication import utility import os, sys,platform, hashlib,time from utility import read_in_chunks def launch(localpath,swift_container,prefix,size_limit_reading_os,download,fail_tries ,md5_compare, encrypted,encrypt_key,excluded_patterns): print ("Localpath " + localpath) print ("Swift container " + swift_container) print ("Swift prefix " + prefix) print ("Os reading Limit " + str(size_limit_reading_os)) print ("Fail tries " + str(fail_tries)) print ("Download " + str(download)) print ("MD5 Compare " + str(md5_compare)) print ("Encrypted " + str(encrypted)) if encrypted: if encrypt_key == None: print("Error Bad Encryption Key") sys.exit(-3) else: print ("Encrypted key " + "hidden") print("___________") #checks utility.check_segments_size_single (size_limit_reading_os) if utility.check_start_slash(swift_container) or utility.check_start_slash(prefix) or not utility.check_end_slash(localpath) or utility.check_end_slash(swift_container) or not utility.check_end_slash(prefix): print ("Errors on local or remote paths. Checks \\ or / at the begin or end of each path") sys.exit(-2) localfiles = utility.get_local_files (localpath) print ("Files locali " + str(len(localfiles))) print("___________") swift_conn = authentication.set_authentication () results = utility.get_list(fail_tries,swift_conn,swift_container,prefix) if not results: return swift_conn,objects = results results = utility.list_compute_correct_size (fail_tries,objects,swift_conn,swift_container,prefix) if not results: return byte0real,byte0manifest,swift_conn,remotefiles,remotefiles_md5,remotefiles_xobj = results remotefiles_encr,list_enc_old = utility.list_compute_correct_names_for_encryption(objects,prefix) print ("Files remoti " + str(len(remotefiles))) difffiles = {} skipped = 0 sizetodownload=0 filestodownload = 0 skipped_old_md5_large_files = 0 if encrypted and download : import utility_aes for rname in remotefiles_encr.values(): download_file = True for pattern in excluded_patterns: if pattern in rname: download_file = False break if download_file: #print("Comparing " + lname) if encrypted and rname.endswith("_xg10v10_encrypted"): rnamedec = rname.split("_xg10v10_")[0] rnamesize = rname.split("_xg10v10_")[2] rnamemd5 = rname.split("_xg10v10_")[1] if rnamedec not in localfiles.keys() or localfiles[rnamedec] != int(rnamesize) : difffiles[rname] = remotefiles[rname] sizetodownload = sizetodownload + remotefiles[rname] filestodownload = filestodownload + 1 elif md5_compare : hash = hashlib.md5() with open(localpath + rnamedec, 'rb') as f: for piece in read_in_chunks(f,size_limit_reading_os): hash.update(piece) if hash.hexdigest() != rnamemd5: difffiles[rname] = remotefiles[rname] sizetodownload = sizetodownload + remotefiles[rname] filestodownload = filestodownload + 1 else: if rname not in localfiles.keys() or localfiles[rname] != remotefiles[rname] : difffiles[rname] = remotefiles[rname] sizetodownload = sizetodownload + remotefiles[rname] filestodownload = filestodownload + 1 elif md5_compare : if remotefiles_md5[rname] != "0": hash = hashlib.md5() with open(localpath + rname, 'rb') as f: for piece in read_in_chunks(f,size_limit_reading_os): hash.update(piece) if hash.hexdigest() != remotefiles_md5[rname] : difffiles[rname] = remotefiles[rname] sizetodownload = sizetodownload + remotefiles[rname] filestodownload = filestodownload + 1 else: skipped_old_md5_large_files = skipped_old_md5_large_files + 1 else: print("Skipped " + rname + " due to filters" ) skipped = skipped + 1 print("___________Differenze___________") print ("Files locali " + str(len(localfiles))) print ("Skipped due to filters " + str(skipped)) if md5_compare: print ("Skipped md5 comparison on old large files without md5 embedded in x-object-manifest: " + str(skipped_old_md5_large_files)) print ("Files remoti " + str(len(remotefiles))) print ("Files remoti - skipped " + str(len(remotefiles) - skipped)) print ("Files remoti 0byte reali " + str(byte0real)) print ("Files remoti 0byte large " + str(byte0manifest)) print("___________Files to download______") for files, sizes in difffiles.items(): if encrypted and files.endswith("_xg10v10_encrypted"): print(files.split("_xg10v10_")[0],sizes) else: print(files,sizes) print("___________") print ("MBytes to download " + str(sizetodownload/1000000)) print ("Files to download " + str(filestodownload)) print("___________") remainingtodownload = sizetodownload if download : errors_download = 0 skipped_downloads = 0 for file, size in difffiles.items(): for fail_tries_counter in range (fail_tries) : try: print("Downloading File: " + utility.dash_replace(prefix + file) + " " + str(size) ) f = swift_conn.get_object(swift_container,utility.dash_replace(prefix + file),resp_chunk_size=size_limit_reading_os) if not os.path.isdir(utility.folder_from_path(localpath + file,utility.set_dash())[:-1]): os.makedirs(utility.folder_from_path(localpath + file,utility.set_dash())[:-1]) with open( localpath + file,'wb') as t: while True: try: t.write(f[1].next()) except StopIteration: break if encrypted and file.endswith("_xg10v10_encrypted"): rnamedec = file.split("_xg10v10_")[0] with open(localpath + file, 'rb') as f: with open(localpath + rnamedec, 'wb') as d: aes = utility_aes.AESCipherFile(f,d, size_limit_reading_os, encrypt_key) aes.decrypt() os.remove(localpath + file) print("Decrypted") remainingtodownload = remainingtodownload - size print("Remaining to download : " + str(remainingtodownload / 1000000) + " MB") except Exception as e: print("Exception during download") print(e) time.sleep(2) errors_download = errors_download + 1 swift_conn = authentication.set_authentication () if fail_tries_counter == fail_tries - 1 : print("Maximum tries reached. Skipping download of the file") skipped_downloads = skipped_downloads + 1 else: break print("___________") print("Download Terminated : Remaining MB to download " + str(remainingtodownload)) print("Errors during download : " + str(errors_download)) print("Skipped files during download : " + str(skipped_downloads)) else: print("Download Disabled") swift_conn.close()