Differences

This shows you the differences between two versions of the page.

--- accounting_troubleshooting [2026/02/04 17:59] – bbruzzo
+++ accounting_troubleshooting [2026/02/06 17:15] (current) – bbruzzo
@@ Line 48: / Line 48: @@
     print(stdout)
-if __name__ == '__main__':
+if __name__ == '__main__':file> syntax as above, you might want to make the shown code available for download as well. You can
     accounts = get_accounts()
@@ Line 295: / Line 295: @@
 ¿Vuelve a entrar el mismo job?
+Si, vuelve a correr:
+<code>[root@mmgt02 ~]# sacct -X -a -q qosprueba --starttime=2026-02-04 --format=JobID,JobName,Account,QOS,AllocCPUS,State,Elapsed
+JobID           JobName    Account        QOS  AllocCPUS      State    Elapsed
+------------ ---------- ---------- ---------- ---------- ---------- ----------
+            sleep       root  qosprueba         12    TIMEOUT   03:20:24
+            sleep       root  qosprueba         12    TIMEOUT   00:52:59
+</code>
+<code>[root@mmgt02 ~]# sacct -X -a -q qosprueba --starttime=2025-01-01 --parsable2 --noheader --format=elapsedraw,ncpus | awk -F'|' '{sum+=$1*$2} END {print sum/60}'
+.28
+</code>
+Vemos que la qos tiene consumidos 4141 minutos.
+Cree que corrió 3040 minutos:
+<code># scontrol show assoc_mgr | grep -A 7 "QOS=qosprueba("
+QOS=qosprueba(32)
+    UsageRaw=182436.000000
+    GrpJobs=N(0) GrpJobsAccrue=N(0) GrpSubmitJobs=N(0) GrpWall=N(253.38)
+    GrpTRES=cpu=N(0),mem=N(0),energy=N(0),node=N(0),billing=N(0),fs/disk=N(0),vmem=N(0),pages=N(0),gres/gpu=N(0),gres/gpu:v100=N(0),gres/gpumem=N(0),gres/gpuutil=N(0)
+    GrpTRESMins=cpu=3000(3040),mem=N(0),energy=N(0),node=N(253),billing=N(3040),fs/disk=N(0),vmem=N(0),pages=N(0),gres/gpu=2000(0),gres/gpu:v100=N(0),gres/gpumem=N(0),gres/gpuutil=N(0)
+    GrpTRESRunMins=cpu=N(0),mem=N(0),energy=N(0),node=N(0),billing=N(0),fs/disk=N(0),vmem=N(0),pages=N(0),gres/gpu=N(0),gres/gpu:v100=N(0),gres/gpumem=N(0),gres/gpuutil=N(0)
+    MaxWallPJ=
+    MaxTRESPJ=
+</code>
@@ Line 306: / Line 333: @@
-To be continued...
+====== FIX ======
+Utilicé este script para actualizar las qos a los valores restantes y setear las qos default:
+<code python fix_qos.py>#!/usr/bin/env python3.10
+import subprocess
+def get_accounts():
+    command = ['sacctmgr', '--noheader', 'list', 'account', 'format=account']
+    output = subprocess.run(command,capture_output=True,encoding='utf-8')
+    accounts = output.stdout.split()
+    return accounts
+def get_hours(account):
+    command = ['sacct', '-X', '-a', '-A', str(account),
+            '--starttime=2025-01-01', '--parsable2', '--noheader',
+            '--format=elapsedraw,ncpus']
+    cpu_hours_command = ['sreport', 'cluster', 'userutilizationbyaccount',
+            '--noheader', '--parsable2', '--tres=cpu',
+            'accounts='+str(account), 'format=used',
+            'start=2025-05-01T00:00:00']
+    gpu_hours_command = ['sreport', 'cluster', 'userutilizationbyaccount',
+            '--noheader', '--parsable2', '--tres=gres/gpu',
+            'accounts='+str(account), 'format=used',
+            'start=2025-05-01T00:00:00']
+    cpu_hours = subprocess.run(
+            cpu_hours_command,
+            capture_output=True,
+            encoding='utf-8').stdout.split()
+    cpu_hours = [int(i) for i in cpu_hours]
+    cpu_hours = sum(cpu_hours)
+    gpu_hours = subprocess.run(
+            gpu_hours_command,
+            capture_output=True,
+            encoding='utf-8').stdout.split()
+    gpu_hours = [int(i) for i in gpu_hours]
+    gpu_hours = sum(gpu_hours)
+    return cpu_hours, gpu_hours
+def update_qos(account):
+    match account:
+        case a if a.startswith('pad'):
+            default_cpu, default_gpu = 240000000,15000000
+        case a if a.startswith('pci'):
+            default_cpu, default_gpu = 60000000,3600000
+        case a if a.startswith('pisca'):
+            default_cpu, default_gpu = 6000000,60000
+    spent_cpu, spent_gpu = get_hours(account)
+    new_cpu = max(0,default_cpu-spent_cpu)
+    new_gpu = max(0,default_gpu-spent_gpu)
+    update_command = ['sacctmgr', 'modify', 'qos', 'qos_'+str(account), 'set',
+            'GrpTRESMins=cpu='+str(new_cpu)+',gres/gpu='+str(new_gpu)]
+    print(update_command)
+    subprocess.run(update_command)
+def update_defqos(account):
+    '''
+    Sets Default QOS to IPAC project in case of misconfiguration.
+    '''
+    command =  ['sacctmgr', 'modify', 'account', 'set',
+            'defaultqos=qos_'+str(account), 'where', 'account='+str(account)]
+    subprocess.run(command)
+def action(accounts,func=get_hours):
+    for account in accounts:
+        if account.startswith(('pad','pci','pisca')):
+            func(account)
+actions = {
+        '1': get_hours,
+        '2': update_defqos,
+        '3': update_qos,
+}
+if __name__ == '__main__':
+    print("Elegir que acción ejecutar en slurm:\n","1) get_hours\n",
+            "2) update_defqos\n","3) update_qos\n")
+    choice = input("Elegir:")
+    func = actions.get(choice)
+    accounts = get_accounts()
+    if func:
+        action(accounts,func)
+    else:
+        print("Operación invalida")
+</code>