1+ import os
2+ import pandas as pd
3+ from datetime import date
4+ from omicspred .models import *
5+ from exports .exports import PGSExport , fields_to_export
6+
7+ exports_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/metadata_exports'
8+ file_url_prefix = 'https://app.box.com/s/'
9+
10+ nested_attr_sep = '__'
11+
12+ def get_data_attr (model :object ,model_type :str )-> dict :
13+ export_data = {}
14+ for field in fields_to_export [model_type ]:
15+ # Skip the ManyToMany relations and the non native fields
16+ if 'skip_auto_import' in field .keys ():
17+ continue
18+ field_name = field ['name' ]
19+ if nested_attr_sep in field_name :
20+ attrs = field_name .split (nested_attr_sep )
21+ nested_model = getattr (model ,attrs [0 ])
22+ value = getattr (nested_model ,attrs [1 ])
23+ export_data [field_name ] = value
24+ else :
25+ export_data [field_name ] = getattr (model , field_name )
26+ # Date format
27+ data_value = export_data [field_name ]
28+ if isinstance (data_value , date ):
29+ export_data [field_name ] = data_value .strftime ('%m/%d/%Y' )
30+ # print(f'{field_name}: {export_data[field_name]} | {type(export_data[field_name])}')
31+ return export_data
32+
33+
34+ def get_molecular_traits (mt_models :list ,mt_type :str ,score_data :dict ) -> dict :
35+ mt_dict = {
36+ 'external_id' : [],
37+ 'name' : []
38+ }
39+ for mt_model in mt_models :
40+ mt_data = get_data_attr (mt_model ,'MolecularTrait' )
41+ for field in mt_dict .keys ():
42+ if mt_data [field ]:
43+ mt_dict [field ].append (mt_data [field ])
44+ for field in mt_dict .keys ():
45+ score_data [f'{ mt_type } { field } ] = ',' .join (mt_dict [field ])
46+ return score_data
47+
48+
49+ def run ():
50+
51+ # datasets = Dataset.objects.all().order_by('num')
52+ datasets = Dataset .objects .filter (num = 1 ).order_by ('num' )
53+ print ("## Start metadata exports, dataset by dataset" )
54+ datasets_total = len (datasets )
55+ count_dataset = 0
56+ for dataset in datasets :
57+ count_dataset += 1
58+ dataset_id = dataset .id
59+ print (f"- Dataset { dataset_id } { count_dataset } { datasets_total } )
60+ # Prepare data for exports
61+ data = {
62+ 'dataset' : get_data_attr (dataset ,'Dataset' ),
63+ 'publication' : get_data_attr (dataset .publication ,'Publication' ),
64+ 'scores' : [],
65+ 'performances' : [],
66+ 'cohorts' : []
67+ }
68+
69+ # /!\ TODO: add files url to dataset
70+ files_ids_dict = dataset .files_ids
71+ for file in files_ids_dict .keys ():
72+ data ['dataset' ][f'file_url_{ file } ] = f'{ file_url_prefix } { files_ids_dict [file ]}
73+
74+ # Prepare cohort data
75+ cohorts_names = set ()
76+ cohorts_data = []
77+ for t_sample in dataset .samples_training .all ():
78+ cohorts = t_sample .cohorts .all ().order_by ('name_short' )
79+ for cohort in cohorts :
80+ if cohort .name_short not in cohorts_names :
81+ cohorts_data .append (get_data_attr (cohort ,'Cohort' ))
82+ cohorts_names .add (cohort .name_short )
83+ for v_sample in dataset .samples_validation .all ():
84+ cohorts = v_sample .cohorts .all ().order_by ('name_short' )
85+ for cohort in cohorts :
86+ if cohort .name_short not in cohorts_names :
87+ cohorts_data .append (get_data_attr (cohort ,'Cohort' ))
88+ cohorts_names .add (cohort .name_short )
89+ data ['cohorts' ] = cohorts_data
90+
91+ # Prepare score data
92+ scores = dataset .dataset_score .all ().order_by ('num' )
93+ for score in scores :
94+ # Prepare score data
95+ score_data = get_data_attr (score ,'Score' )
96+ # Genes
97+ score_data = get_molecular_traits (score .genes .all (),'genes' ,score_data )
98+ # Proteins
99+ score_data = get_molecular_traits (score .proteins .all (),'proteins' ,score_data )
100+ # Metabolites
101+ score_data = get_molecular_traits (score .metabolites .all (),'metabolites' ,score_data )
102+
103+ data ['scores' ].append (score_data )
104+
105+ # Prepare performances (metrics, samples, cohorts)
106+ for perf in score .score_performance .all ().order_by ('id' ):
107+ sample_perf_data = get_data_attr (perf ,'Performance' )
108+ # Update eval_type using the long name
109+ sample_perf_data ['eval_type' ] = perf .get_eval_type_display ()
110+
111+ # Cohorts
112+ cohorts_list = []
113+ for cohort in perf .sample .cohorts .all ():
114+ cohorts_list .append (cohort .name_short )
115+ sample_perf_data ['cohorts' ] = ',' .join (sorted (cohorts_list ))
116+
117+ # Metrics
118+ metrics_data = {
119+ 'metrics_r2' : None ,
120+ 'metrics_r2_pval' : None ,
121+ 'metrics_rho' : None ,
122+ 'metrics_rho_pval' : None ,
123+ 'metrics_match_rate' : None
124+ }
125+ for metric in perf .performance_metric .all ():
126+ m_name = metric .name_short
127+ if m_name == 'R2' :
128+ metrics_data ['metrics_r2' ] = metric .estimate
129+ metrics_data ['metrics_r2_pval' ] = metric .pvalue
130+ elif m_name == 'Rho' :
131+ metrics_data ['metrics_rho' ] = metric .estimate
132+ metrics_data ['metrics_rho_pval' ] = metric .pvalue
133+ elif m_name == 'Match Rate' :
134+ metrics_data ['metrics_match_rate' ] = metric .estimate
135+ if sample_perf_data ['eval_type' ] == 'Training' :
136+ metrics_data ['metrics_match_rate' ] = 1
137+ for md in metrics_data .keys ():
138+ sample_perf_data [md ] = metrics_data [md ]
139+
140+ data ['performances' ].append (sample_perf_data )
141+
142+ # for dtype in data.keys():
143+ # print(f'# {dtype}:\n{data[dtype]}')
144+ # exit()
145+
146+ # Create export
147+ filename = f'{ exports_dir } { dataset_id }
148+ op_export = PGSExport (filename , data , dataset_id )
149+ op_export .generate_sheets ()
150+ op_export .save ()
151+ # print(data)
0 commit comments