Commit d5398a86 authored by Mitchell Moore's avatar Mitchell Moore
Browse files

Fix var shadowing of set.

parents 12b39f21 33ba2106
......@@ -16,8 +16,8 @@ def get_args():
"""
parser = argparse.ArgumentParser(description='Capture Globus transfer speeds between two endpoints.')
parser.add_argument('client', type=str, help='The Client ID of the account testing the transfers')
parser.add_argument('src_ep_id', type=str, help='The Endpoint ID of the source endpoint.')
parser.add_argument('dest_ep_id', type=str, help='The Endpoint ID of the destination endpoint.')
# parser.add_argument('src_ep_id', type=str, help='The Endpoint ID of the source endpoint.')
# parser.add_argument('dest_ep_id', type=str, help='The Endpoint ID of the destination endpoint.')
parser.add_argument('src_dir', type=str, help='The desired directory from the source endpoint.')
parser.add_argument('dest_dir', type=str, help='The desired directory from the destination endpoint.')
parser.add_argument('-t, --refresh-token', dest='refresh_token', type=str,
......@@ -31,6 +31,7 @@ def get_args():
help='Use to delete/clean the transferred files post transfer.')
parser.add_argument('-b', dest='batch', action='store_true',
help='Run all transfers at the same time rather than individually.')
parser.add_argument('-l', dest='list', choices=['01', '04', '06', '08', '10', '12', '14', '16'], nargs="*", help= 'Run a specific list of datasets.')
return parser.parse_args()
......@@ -138,10 +139,10 @@ def single_transfer_data(tc, src_id, dest_id, src_dir, dest_dir, output):
exit(-1)
def multi_transfer(tc, src_id, dest_id, src_dir, dest_dir, output):
def multi_transfer(tc, src_id, dest_id, src_dir, dest_dir, output, **kwargs):
"""
Will initiate the transfer of a multiple datasets (ds01-ds16) and show the progress in the console. Will submit all
transfers as a batch given the "-b" flag in the console.
transfers as an asynchronous batch given the "-b" flag in the console.
@param tc: (globus_sdk.transfer.client.TransferClient) Client to conduct the transfer.
@param src_id: (string) Globus endpoint id of the Source endpoint.
@param dest_id: (string) Globus endpoint id of the Destination endpoint.
......@@ -151,7 +152,8 @@ def multi_transfer(tc, src_id, dest_id, src_dir, dest_dir, output):
@return: (dictionary) Object that contains transfer details and results.
"""
data_sets = ["01", "04", "06", "08", "10", "12", "14", "16"]
data_sets = args.list if args.list is not None else ["01", "04", "06", "08", "10", "12", "14", "16"]
if args.batch:
ready_transfers = []
......@@ -206,24 +208,53 @@ def write_results(data_dict, filename):
data_dict["task_id"]])
def clean():
def multiple_endpoints():
ch_on = '9c8c88c2-ea4a-11e6-b9ba-22000b9a448b'
ch_off = '7167cb38-9f78-11e6-b0dd-22000b92c261'
cas = '606579ae-5b03-11e9-bf32-0edbf3a4e7ee'
argonne = 'e261ffb8-6d04-11e5-ba46-22000b92c6ec'
pam = '924a32b0-6a2a-11e6-83a8-22000b97daec'
# On-Campus
multi_transfer(tc, cas, ch_on, args.src_dir, args.dest_dir, True) # cas
multi_transfer(tc, argonne, ch_on, args.src_dir, args.dest_dir, True) # argonne
# multi_transfer(tc, pam, ch_on, "/globus{}".format(args.src_dir), args.dest_dir, True, hack="/globus/perftest/uab_rc/") # pam
# Off-Campus
multi_transfer(tc, cas, ch_off, args.src_dir, args.dest_dir, True) # cas
multi_transfer(tc, argonne, ch_off, args.src_dir, args.dest_dir, True) # argonne
# multi_transfer(tc, pam, ch_off, "/globus{}".format(args.src_dir), args.dest_dir, True, hack="/globus/perftest/uab_rc/") # pam
return [ch_on, ch_off, cas, argonne]
def clean(ep_list):
"""
Deletes the written data from the endpoints.
@return: None
"""
if args.clean:
ddata = globus_sdk.DeleteData(tc, args.dest_ep_id, recursive=True,
label="Delete {}".format(args.dest_dir.split("/")[-1]))
for item in args.list:
dest_dir = args.dest_dir + "ds{}".format(item)
ddata = globus_sdk.DeleteData(tc, ep_list[1], recursive=True,
label="Delete {}".format(dest_dir.split("/")[-1]))
ddata.add_item(args.dest_dir)
tc.submit_delete(ddata)
ddata.add_item(dest_dir)
tc.submit_delete(ddata)
if args.write_back and (args.return_directory is not None): # Ensure that data is not deleted from host.
ddir = args.return_directory or args.src_dir
ddata2 = globus_sdk.DeleteData(tc, args.src_ep_id, recursive=True,
label="Delete {}".format(args.src_dir.split("/")[-1]))
ddata2.add_item(ddir)
tc.submit_delete(ddata2)
for ep in ep_list[2:]: # Delete transferred datasets from other endpoints
for item in args.list:
src_dir = args.src_dir + "ds{}".format(item)
ddir = args.return_directory + "ds{}".format(item) or src_dir
ddata2 = globus_sdk.DeleteData(tc, ep, recursive=True,
label="Delete {}".format(ddir.split("/")[-1]))
ddata2.add_item(ddir)
tc.submit_delete(ddata2)
if __name__ == '__main__':
......@@ -234,10 +265,6 @@ if __name__ == '__main__':
test_start = datetime.now().strftime("%m-%d-%Y_%Hh%Mm%Ss")
<<<<<<< Updated upstream
multi_transfer(tc, args.src_ep_id, args.dest_ep_id, args.src_dir, args.dest_dir, True)
clean()
=======
# multi_transfer(tc, args.src_ep_id, args.dest_ep_id, args.src_dir, args.dest_dir, True)
# clean()
eps = multiple_endpoints() # hardcoded ep values
......@@ -246,4 +273,3 @@ if __name__ == '__main__':
# *** Example run ***
# python generate_data.py $CLIENT_ID /datasets/ /data/user/mmoo97/TEST_TRANSFERS/ -l
# 01 -t $TOKEN -r /perftest/uab_rc/ -wc
>>>>>>> Stashed changes
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment