diff --git a/docker/rucio_client/scripts/CMSRSE.py b/docker/rucio_client/scripts/CMSRSE.py index 5ace5b3a..3f2a4eab 100644 --- a/docker/rucio_client/scripts/CMSRSE.py +++ b/docker/rucio_client/scripts/CMSRSE.py @@ -30,6 +30,10 @@ 'wan': {'read': 1, 'write': 1, 'third_party_copy_write': 1, 'third_party_copy_read': 1, 'delete': 1}, 'lan': {'read': None, 'write': None, 'delete': None}}, + 'user': { + 'wan': {'read': 1, 'write': 1, 'third_party_copy_write': 1, 'third_party_copy_read': 1, + 'delete': 1}, + 'lan': {'read': None, 'write': None, 'delete': None}}, } RUCIO_PROTOS = ['SRMv2', 'XRootD', 'WebDAV'] PROTO_WEIGHT_TPC = {'WebDAV': 1, 'XRootD': 3, 'SRMv2': 2} @@ -64,17 +68,21 @@ def __init__(self, json, dry=False, cms_type='real', deterministic=True): xattrs = {} - # If we are building a _Test or _Temp instance add the special prefix + # If we are building a _Test, _Temp, or _User instance add the special suffix if cms_type == "test": self.rse_name = json['rse']+"_Test" elif cms_type == "temp": self.rse_name = json['rse']+"_Temp" + elif cms_type == "user": + self.rse_name = json['rse']+"_User" else: self.rse_name = json['rse'] if json.get('loadtest', None) is not None: xattrs['loadtest'] = json['loadtest'] - xattrs['fts'] = ','.join(json['fts']) + # Provide default FTS list if not specified in JSON + fts_list = json.get('fts') or ["https://fts3-cms.cern.ch:8446"] + xattrs['fts'] = ','.join(fts_list) self._get_attributes(xattrs=xattrs) """ diff --git a/docker/rucio_client/scripts/setOneRucioFromGitlab b/docker/rucio_client/scripts/setOneRucioFromGitlab index 69f71d8e..db9f259e 100755 --- a/docker/rucio_client/scripts/setOneRucioFromGitlab +++ b/docker/rucio_client/scripts/setOneRucioFromGitlab @@ -15,7 +15,7 @@ import logging import sys parser = ArgumentParser(description="Update a site definition from GitLab") -parser.add_argument('--type', dest='cms_type', help='type of RSE (prod-real, int-real, test, temp).',default=None, required=True) +parser.add_argument('--type', dest='cms_type', help='type of RSE (prod-real, int-real, test, temp, user).',default=None, required=True) parser.add_argument('--dryrun', dest='dry_run', action='store_true') parser.add_argument('--debug', action='store_true', help='be more verbose') parser.add_argument('--just_print', dest='print_scheme', help='just prints the given scheme: gsiftp, srm, davs, xroot, all', \ diff --git a/docker/rucio_client/scripts/setRucioFromGitlab b/docker/rucio_client/scripts/setRucioFromGitlab index 70dbbb4d..dcf1e8b9 100755 --- a/docker/rucio_client/scripts/setRucioFromGitlab +++ b/docker/rucio_client/scripts/setRucioFromGitlab @@ -14,8 +14,11 @@ from CMSRSE import CMSRSE SKIP_SITES = [] +# PoC hardcoded list of RSE names for which _User variants should be created +USER_TIER_POC_SITES = ['T2_IT_Rome'] + parser = ArgumentParser(description="Update a site definition from GitLab") -parser.add_argument('--type', dest='cms_type', help='type of RSE (prod-real, int-real, test, temp).', +parser.add_argument('--type', dest='cms_type', help='type of RSE (prod-real, int-real, test, temp, user).', default=None, required=True) parser.add_argument('--dryrun', dest='dry_run', action='store_true', help='do not change anything in rucio, checking only') @@ -59,7 +62,7 @@ for project in projects: continue print(f'Checking {site["rse"]} and type {options.cms_type}') if site['rse'] and options.cms_type in ['test', 'temp']: - # For these, query the actual site and construct a JSON + # For test and temp types, apply to all RSEs if options.cms_type == 'test': rse_name = site['rse'] + '_Test' deterministic = True @@ -70,8 +73,28 @@ for project in projects: print(' Skipping.') continue - if 'fts' not in site: - site.update({'fts': ["https://fts3-cms.cern.ch:8446", "https://lcgfts3.gridpp.rl.ac.uk:8446"]}) + rse = CMSRSE(site, dry=options.dry_run, cms_type=options.cms_type, deterministic=deterministic) + try: + if rse.update(): + print(f'RSE {rse.rse_name} and type {rse.rucio_rse_type} changed') + else: + print(f'RSE {rse.rse_name} and type {rse.rucio_rse_type} unchanged') + except Exception: + print(f'Could not update RSE {rse.rse_name}. Traceback:') + print(traceback.format_exc()) + elif site['rse'] and options.cms_type == 'user': + # For user type, only create _User variants for PoC sites + if site['rse'] not in USER_TIER_POC_SITES: + print(f' Skipping (not in PoC list: {USER_TIER_POC_SITES})') + continue + # For these, query the actual site and construct a JSON + rse_name = site['rse'] + '_User' + deterministic = True + # Allow PoC _User RSEs to be created even if not yet in Rucio + if rse_name in SKIP_SITES: + print(' Skipping.') + continue + rse = CMSRSE(site, dry=options.dry_run, cms_type=options.cms_type, deterministic=deterministic) try: if rse.update(): diff --git a/docker/rucio_client/scripts/setSiteCapacity b/docker/rucio_client/scripts/setSiteCapacity index 953bea7e..744d3fcb 100755 --- a/docker/rucio_client/scripts/setSiteCapacity +++ b/docker/rucio_client/scripts/setSiteCapacity @@ -114,7 +114,8 @@ try: logger.warn(f"Unexpected value for default_min_free_space_percentage: {e}") default_min_free_space_percentage = DEFAULT_MIN_FREE_PERCENTAGE - rses = [rse['rse'] for rse in rclient.list_rses('cms_type=real&rse_type=DISK')] + # Query both production (cms_type=real) and user tier (cms_type=user) DISK RSEs + rses = [rse['rse'] for rse in rclient.list_rses('(cms_type=real|cms_type=user)&rse_type=DISK')] tape_rses = [rse['rse'] for rse in rclient.list_rses('cms_type=real&rse_type=TAPE')] for site in sites : @@ -145,62 +146,83 @@ try: logger.debug(f"RSE {site['name']} or {site['name']}_Disk not a valid RSE") continue - if rse in skip_rses: - logger.debug(f"Skipping static usage update for {rse}") - continue - - logger.debug(f"Updating static usage for {rse}") - disk_experiment_use_bytes = site['disk_experiment_use'] * 1e12 #total space used by CMS experiment data - disk_local_use_bytes = site['disk_local_use'] * 1e12 #additional quota for local rse_local_users account - rse_available_bytes = disk_experiment_use_bytes + disk_local_use_bytes - - # Set Quota for local users account for the rse - setLocalUsersQuota(rclient, group_accounts, rse, disk_local_use_bytes, logger, dry_run) - - min_free_space_percentage = default_min_free_space_percentage - rse_attributes = rclient.list_rse_attributes(rse=rse) - if 'min_free_space_percentage' in rse_attributes: - min_free_space_percentage = float(rse_attributes['min_free_space_percentage']) - if min_free_space_percentage > MAX_MIN_FREE_PERCENTAGE: - min_free_space_bytes = default_min_free_space_percentage - - try: - # Static usage is 0 for many T3s - These are managed as quasi-static - if disk_experiment_use_bytes == 0: - logger.debug(f"Static usage for {rse} is 0, skipping") + # Process main RSE + rses_to_update = [rse] + + # Also process _User variant if it exists + user_rse = f"{rse.replace('_Disk', '')}_User" + if user_rse in rses: + rses_to_update.append(user_rse) + + for rse in rses_to_update: + if rse in skip_rses: + logger.debug(f"Skipping static usage update for {rse}") continue - current_static_usage = list(rclient.get_rse_usage(rse=rse, filters={'source':'static'})) - # Taking into account the case where the static usage is not yet set but the site wishes to change from quasi-static to static - if len(current_static_usage) == 0: - current_static_usage = 0 + logger.debug(f"Updating static usage for {rse}") + disk_experiment_use_bytes = site['disk_experiment_use'] * 1e12 #total space used by CMS experiment data + disk_local_use_bytes = site['disk_local_use'] * 1e12 #additional quota for local rse_local_users account + + # For _User suffix RSEs, the entire capacity comes from disk_local_use + if rse.endswith('_User'): + rse_available_bytes = disk_local_use_bytes else: - current_static_usage = current_static_usage[0]['used'] + rse_available_bytes = disk_experiment_use_bytes + disk_local_use_bytes + + # Set Quota for local users account for the rse (skip for _User suffix RSEs) + if not rse.endswith('_User'): + setLocalUsersQuota(rclient, group_accounts, rse, disk_local_use_bytes, logger, dry_run) + + min_free_space_percentage = default_min_free_space_percentage + rse_attributes = rclient.list_rse_attributes(rse=rse) + if 'min_free_space_percentage' in rse_attributes: + min_free_space_percentage = float(rse_attributes['min_free_space_percentage']) + if min_free_space_percentage > MAX_MIN_FREE_PERCENTAGE: + min_free_space_bytes = default_min_free_space_percentage + + try: + # Static usage is 0 for many T3s - These are managed as quasi-static + # For _User suffix RSEs, check disk_local_use instead of disk_experiment_use + if rse.endswith('_User'): + skip_check = disk_local_use_bytes == 0 + else: + skip_check = disk_experiment_use_bytes == 0 + + if skip_check: + logger.debug(f"Static usage for {rse} is 0, skipping") + continue + + current_static_usage = list(rclient.get_rse_usage(rse=rse, filters={'source':'static'})) + # Taking into account the case where the static usage is not yet set but the site wishes to change from quasi-static to static + if len(current_static_usage) == 0: + current_static_usage = 0 + else: + current_static_usage = current_static_usage[0]['used'] - rse_limits = rclient.get_rse_limits(rse=rse) - if 'MinFreeSpace' in rse_limits: - current_min_free_space = rse_limits['MinFreeSpace'] - else: - current_min_free_space = 0 + rse_limits = rclient.get_rse_limits(rse=rse) + if 'MinFreeSpace' in rse_limits: + current_min_free_space = rse_limits['MinFreeSpace'] + else: + current_min_free_space = 0 - min_free_space_bytes = int(rse_available_bytes*min_free_space_percentage*0.01) - # Trigger update on both value and configuraton change - if current_static_usage == rse_available_bytes and current_min_free_space == min_free_space_bytes: - logger.debug(f"Static usage for {rse} already up to date") - continue + min_free_space_bytes = int(rse_available_bytes*min_free_space_percentage*0.01) + # Trigger update on both value and configuraton change + if current_static_usage == rse_available_bytes and current_min_free_space == min_free_space_bytes: + logger.debug(f"Static usage for {rse} already up to date") + continue - if dry_run: - logger.info(f"Updating static usage, from {current_static_usage*1e-12:.2f}TB to {rse_available_bytes*1e-12:.2f}TB, for {rse}, dry_run=True") - logger.info(f"Updating MinFreeSpace, from {current_min_free_space*1e-12:.2f}TB to {min_free_space_bytes*1e-12:.2f}TB, for {rse}, dry_run=True") + if dry_run: + logger.info(f"Updating static usage, from {current_static_usage*1e-12:.2f}TB to {rse_available_bytes*1e-12:.2f}TB, for {rse}, dry_run=True") + logger.info(f"Updating MinFreeSpace, from {current_min_free_space*1e-12:.2f}TB to {min_free_space_bytes*1e-12:.2f}TB, for {rse}, dry_run=True") - else: - rclient.set_rse_usage(rse=rse, source='static', used=rse_available_bytes, free=None) - rclient.set_rse_limits(rse=rse, name='MinFreeSpace', value=min_free_space_bytes) - logger.info(f"Updating static usage, from {current_static_usage*1e-12:.2f}TB to {rse_available_bytes*1e-12:.2f}TB, for {rse}") - logger.info(f"Updating MinFreeSpace, from {current_min_free_space*1e-12:.2f}TB to {min_free_space_bytes*1e-12:.2f}TB, for {rse}") - except Exception as e: - logger.error(f"Failed to update static usage for {rse}: {e}") - traceback.print_exc() + else: + rclient.set_rse_usage(rse=rse, source='static', used=rse_available_bytes, free=None) + rclient.set_rse_limits(rse=rse, name='MinFreeSpace', value=min_free_space_bytes) + logger.info(f"Updating static usage, from {current_static_usage*1e-12:.2f}TB to {rse_available_bytes*1e-12:.2f}TB, for {rse}") + logger.info(f"Updating MinFreeSpace, from {current_min_free_space*1e-12:.2f}TB to {min_free_space_bytes*1e-12:.2f}TB, for {rse}") + except Exception as e: + logger.error(f"Failed to update static usage for {rse}: {e}") + traceback.print_exc() except Exception as e: