Skip to content

Commit c54178f

Browse files
ens-lgilens-lgil
authored
Merge pull request #64 from ens-lgil/main
Updates on imports, search, REST API optimisation and OP models
2 parents 6df2a13 + dd501f3 commit c54178f

22 files changed

Lines changed: 784 additions & 548 deletions

‎imports/README.md‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,8 +8,9 @@ python manage.py runscript import_metadata
88
```
99

1010
## Post-processing
11-
1. get_score_ancestry_dist.py
12-
2. update_ensembl_genes.py
13-
3. update_ensembl_proteins.py
14-
4. download_reactome_data.py
15-
5. get_reactome_mappings.py
11+
1. metadata_post_proccessing.py
12+
2. get_score_ancestry_dist.py
13+
3. update_ensembl_genes.py
14+
4. update_ensembl_proteins.py
15+
5. download_reactome_data.py (optional - only for updated dataset from Reactome)
16+
6. get_reactome_mappings.py

‎imports/scripts/generate_genetic_score_files.py‎

Lines changed: 79 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import os, io, sys
2+
import re
23
import sqlite3
34
import pathlib
45
import zipfile
@@ -29,77 +30,98 @@
2930
# | ENSG00000261456.5 | rs11252546 | chr10_58487_T_C_b38 | T | C | -0.0335449590056556 |
3031

3132

32-
pmid = 26258848
33+
pmid = 32913098
34+
ds_name = '- sQTL - Enet -'
3335

34-
dbs_location = '/Users/lg10/Workspace/datafiles/OmicsPred/GTEx_V8/elastic_net_models/'
35-
scores_root_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/GTEx_V8/genetic_scores'
36+
# Enet eQTL
37+
# dbs_location = '/Users/lg10/Workspace/datafiles/OmicsPred/GTEx_V8/elastic_net_models/'
38+
# scores_root_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/GTEx_V8/genetic_scores'
3639

40+
# Enet sQTL
41+
dbs_location = '/Users/lg10/Workspace/datafiles/OmicsPred/GTEx_V8/elastic_net_models_splicing/'
42+
scores_root_dir = '/Users/lg10/Workspace/datafiles/OmicsPred/GTEx_V8/genetic_scores_enet_splicing'
43+
44+
# datasets_list = {
45+
# 'Adipose_Subcutaneous': {'tissue_id': 'UBERON_0002190', 'db':'gtex_splicing_v8_eur_Adipose_Subcutaneous_signif.db'}
46+
# }
3747

3848
datasets_list = {
39-
'Adipose_Subcutaneous': {'tissue_id': 'UBERON_0002190', 'db':'en_Adipose_Subcutaneous.db'},
40-
'Adipose_Visceral_Omentum': {'tissue_id': 'UBERON_0010414', 'db':'en_Adipose_Visceral_Omentum.db'},
41-
'Adrenal_Gland': {'tissue_id': 'UBERON_0002369', 'db':'en_Adrenal_Gland.db'},
42-
'Artery_Aorta': {'tissue_id': 'UBERON_0001496', 'db':'en_Artery_Aorta.db'},
43-
'Artery_Coronary': {'tissue_id': 'UBERON_0001621', 'db':'en_Artery_Coronary.db'},
44-
'Artery_Tibial': {'tissue_id': 'UBERON_0007610', 'db':'en_Artery_Tibial.db'},
45-
'Brain_Amygdala': {'tissue_id': 'UBERON_0001876', 'db':'en_Brain_Amygdala.db'},
46-
'Brain_Anterior_cingulate_cortex_BA24': {'tissue_id': 'UBERON_0009835', 'db':'en_Brain_Anterior_cingulate_cortex_BA24.db'},
47-
'Brain_Caudate_basal_ganglia': {'tissue_id': 'UBERON_0001873', 'db':'en_Brain_Caudate_basal_ganglia.db'},
48-
'Brain_Cerebellar_Hemisphere': {'tissue_id': 'UBERON_0002245', 'db':'en_Brain_Cerebellar_Hemisphere.db'},
49-
'Brain_Cerebellum': {'tissue_id': 'UBERON_0002037', 'db':'en_Brain_Cerebellum.db'},
50-
'Brain_Cortex': {'tissue_id': 'UBERON_0001870', 'db':'en_Brain_Cortex.db'},
51-
'Brain_Frontal_Cortex_BA9': {'tissue_id': 'UBERON_0009834', 'db':'en_Brain_Frontal_Cortex_BA9.db'},
52-
'Brain_Hippocampus': {'tissue_id': 'UBERON_0001954', 'db':'en_Brain_Hippocampus.db'},
53-
'Brain_Hypothalamus': {'tissue_id': 'UBERON_0001898', 'db':'en_Brain_Hypothalamus.db'},
54-
'Brain_Nucleus_accumbens_basal_ganglia': {'tissue_id': 'UBERON_0001882', 'db':'en_Brain_Nucleus_accumbens_basal_ganglia.db'},
55-
'Brain_Putamen_basal_ganglia': {'tissue_id': 'UBERON_0001874', 'db':'en_Brain_Putamen_basal_ganglia.db'},
56-
'Brain_Spinal_cord_cervical_c-1': {'tissue_id': 'UBERON_0006469', 'db':'en_Brain_Spinal_cord_cervical_c-1.db'},
57-
'Brain_Substantia_nigra': {'tissue_id': 'UBERON_0002038', 'db':'en_Brain_Substantia_nigra.db'},
58-
'Breast_Mammary_Tissue': {'tissue_id': 'UBERON_0008367', 'db':'en_Breast_Mammary_Tissue.db'},
59-
'Cells_Cultured_fibroblasts': {'tissue_id': 'EFO_0002009', 'db':'en_Cells_Cultured_fibroblasts.db'},
60-
'Cells_EBV-transformed_lymphocytes': {'tissue_id': 'EFO_0000572', 'db':'en_Cells_EBV-transformed_lymphocytes.db'},
61-
'Colon_Sigmoid': {'tissue_id': 'UBERON_0001159', 'db':'en_Colon_Sigmoid.db'},
62-
'Colon_Transverse': {'tissue_id': 'UBERON_0001157', 'db':'en_Colon_Transverse.db'},
63-
'Esophagus_Gastroesophageal_Junction': {'tissue_id': 'UBERON_0004550', 'db':'en_Esophagus_Gastroesophageal_Junction.db'},
64-
'Esophagus_Mucosa': {'tissue_id': 'UBERON_0006920', 'db':'en_Esophagus_Mucosa.db'},
65-
'Esophagus_Muscularis': {'tissue_id': 'UBERON_0004648', 'db':'en_Esophagus_Muscularis.db'},
66-
'Heart_Atrial_Appendage': {'tissue_id': 'UBERON_0006631', 'db':'en_Heart_Atrial_Appendage.db'},
67-
'Heart_Left_Ventricle': {'tissue_id': 'UBERON_0006566', 'db':'en_Heart_Left_Ventricle.db'},
68-
'Kidney_Cortex': {'tissue_id': 'UBERON_0001225', 'db':'en_Kidney_Cortex.db'},
69-
'Liver': {'tissue_id': 'UBERON_0001114', 'db':'en_Liver.db'},
70-
'Lung': {'tissue_id': 'UBERON_0008952', 'db':'en_Lung.db'},
71-
'Minor_Salivary_Gland': {'tissue_id': 'UBERON_0006330', 'db':'en_Minor_Salivary_Gland.db'},
72-
'Muscle_Skeletal': {'tissue_id': 'UBERON_0011907', 'db':'en_Muscle_Skeletal.db'},
73-
'Nerve_Tibial': {'tissue_id': 'UBERON_0001323', 'db':'en_Nerve_Tibial.db'},
74-
'Ovary': {'tissue_id': 'UBERON_0000992', 'db':'en_Ovary.db'},
75-
'Pancreas': {'tissue_id': 'UBERON_0001150', 'db':'en_Pancreas.db'},
76-
'Pituitary': {'tissue_id': 'UBERON_0000007', 'db':'en_Pituitary.db'},
77-
'Prostate': {'tissue_id': 'UBERON_0002367', 'db':'en_Prostate.db'},
78-
'Skin_Not_Sun_Exposed_Suprapubic': {'tissue_id': 'UBERON_0036149', 'db':'en_Skin_Not_Sun_Exposed_Suprapubic.db'},
79-
'Skin_Sun_Exposed_Lower_leg': {'tissue_id': 'UBERON_0004264', 'db':'en_Skin_Sun_Exposed_Lower_leg.db'},
80-
'Small_Intestine_Terminal_Ileum': {'tissue_id': 'UBERON_0001211', 'db':'en_Small_Intestine_Terminal_Ileum.db'},
81-
'Spleen': {'tissue_id': 'UBERON_0002106', 'db':'en_Spleen.db'},
82-
'Stomach': {'tissue_id': 'UBERON_0000945', 'db':'en_Stomach.db'},
83-
'Testis': {'tissue_id': 'UBERON_0000473', 'db':'en_Testis.db'},
84-
'Thyroid': {'tissue_id': 'UBERON_0002046', 'db':'en_Thyroid.db'},
85-
'Uterus': {'tissue_id': 'UBERON_0000995', 'db':'en_Uterus.db'},
86-
'Vagina': {'tissue_id': 'UBERON_0000996', 'db':'en_Vagina.db'},
87-
'Whole_Blood': {'tissue_id': 'UBERON_0013756', 'db':'en_Whole_Blood.db'}
49+
'Adipose_Subcutaneous': {'tissue_id': 'UBERON_0002190', 'db':'gtex_splicing_v8_eur_Adipose_Subcutaneous_signif.db'},
50+
'Adipose_Visceral_Omentum': {'tissue_id': 'UBERON_0010414', 'db':'gtex_splicing_v8_eur_Adipose_Visceral_Omentum_signif.db'},
51+
'Adrenal_Gland': {'tissue_id': 'UBERON_0002369', 'db':'gtex_splicing_v8_eur_Adrenal_Gland_signif.db'},
52+
'Artery_Aorta': {'tissue_id': 'UBERON_0001496', 'db':'gtex_splicing_v8_eur_Artery_Aorta_signif.db'},
53+
'Artery_Coronary': {'tissue_id': 'UBERON_0001621', 'db':'gtex_splicing_v8_eur_Artery_Coronary_signif.db'},
54+
'Artery_Tibial': {'tissue_id': 'UBERON_0007610', 'db':'gtex_splicing_v8_eur_Artery_Tibial_signif.db'},
55+
'Brain_Amygdala': {'tissue_id': 'UBERON_0001876', 'db':'gtex_splicing_v8_eur_Brain_Amygdala_signif.db'},
56+
'Brain_Anterior_cingulate_cortex_BA24': {'tissue_id': 'UBERON_0009835', 'db':'gtex_splicing_v8_eur_Brain_Anterior_cingulate_cortex_BA24_signif.db'},
57+
'Brain_Caudate_basal_ganglia': {'tissue_id': 'UBERON_0001873', 'db':'gtex_splicing_v8_eur_Brain_Caudate_basal_ganglia_signif.db'},
58+
'Brain_Cerebellar_Hemisphere': {'tissue_id': 'UBERON_0002245', 'db':'gtex_splicing_v8_eur_Brain_Cerebellar_Hemisphere_signif.db'},
59+
'Brain_Cerebellum': {'tissue_id': 'UBERON_0002037', 'db':'gtex_splicing_v8_eur_Brain_Cerebellum_signif.db'},
60+
'Brain_Cortex': {'tissue_id': 'UBERON_0001870', 'db':'gtex_splicing_v8_eur_Brain_Cortex_signif.db'},
61+
'Brain_Frontal_Cortex_BA9': {'tissue_id': 'UBERON_0009834', 'db':'gtex_splicing_v8_eur_Brain_Frontal_Cortex_BA9_signif.db'},
62+
'Brain_Hippocampus': {'tissue_id': 'UBERON_0001954', 'db':'gtex_splicing_v8_eur_Brain_Hippocampus_signif.db'},
63+
'Brain_Hypothalamus': {'tissue_id': 'UBERON_0001898', 'db':'gtex_splicing_v8_eur_Brain_Hypothalamus_signif.db'},
64+
'Brain_Nucleus_accumbens_basal_ganglia': {'tissue_id': 'UBERON_0001882', 'db':'gtex_splicing_v8_eur_Brain_Nucleus_accumbens_basal_ganglia_signif.db'},
65+
'Brain_Putamen_basal_ganglia': {'tissue_id': 'UBERON_0001874', 'db':'gtex_splicing_v8_eur_Brain_Putamen_basal_ganglia_signif.db'},
66+
'Brain_Spinal_cord_cervical_c-1': {'tissue_id': 'UBERON_0006469', 'db':'gtex_splicing_v8_eur_Brain_Spinal_cord_cervical_c-1_signif.db'},
67+
'Brain_Substantia_nigra': {'tissue_id': 'UBERON_0002038', 'db':'gtex_splicing_v8_eur_Brain_Substantia_nigra_signif.db'},
68+
'Breast_Mammary_Tissue': {'tissue_id': 'UBERON_0008367', 'db':'gtex_splicing_v8_eur_Breast_Mammary_Tissue_signif.db'},
69+
'Cells_Cultured_fibroblasts': {'tissue_id': 'EFO_0002009', 'db':'gtex_splicing_v8_eur_Cells_Cultured_fibroblasts_signif.db'},
70+
'Cells_EBV-transformed_lymphocytes': {'tissue_id': 'EFO_0000572', 'db':'gtex_splicing_v8_eur_Cells_EBV-transformed_lymphocytes_signif.db'},
71+
'Colon_Sigmoid': {'tissue_id': 'UBERON_0001159', 'db':'gtex_splicing_v8_eur_Colon_Sigmoid_signif.db'},
72+
'Colon_Transverse': {'tissue_id': 'UBERON_0001157', 'db':'gtex_splicing_v8_eur_Colon_Transverse_signif.db'},
73+
'Esophagus_Gastroesophageal_Junction': {'tissue_id': 'UBERON_0004550', 'db':'gtex_splicing_v8_eur_Esophagus_Gastroesophageal_Junction_signif.db'},
74+
'Esophagus_Mucosa': {'tissue_id': 'UBERON_0006920', 'db':'gtex_splicing_v8_eur_Esophagus_Mucosa_signif.db'},
75+
'Esophagus_Muscularis': {'tissue_id': 'UBERON_0004648', 'db':'gtex_splicing_v8_eur_Esophagus_Muscularis_signif.db'},
76+
'Heart_Atrial_Appendage': {'tissue_id': 'UBERON_0006631', 'db':'gtex_splicing_v8_eur_Heart_Atrial_Appendage_signif.db'},
77+
'Heart_Left_Ventricle': {'tissue_id': 'UBERON_0006566', 'db':'gtex_splicing_v8_eur_Heart_Left_Ventricle_signif.db'},
78+
'Kidney_Cortex': {'tissue_id': 'UBERON_0001225', 'db':'gtex_splicing_v8_eur_Kidney_Cortex_signif.db'},
79+
'Liver': {'tissue_id': 'UBERON_0001114', 'db':'gtex_splicing_v8_eur_Liver_signif.db'},
80+
'Lung': {'tissue_id': 'UBERON_0008952', 'db':'gtex_splicing_v8_eur_Lung_signif.db'},
81+
'Minor_Salivary_Gland': {'tissue_id': 'UBERON_0006330', 'db':'gtex_splicing_v8_eur_Minor_Salivary_Gland_signif.db'},
82+
'Muscle_Skeletal': {'tissue_id': 'UBERON_0011907', 'db':'gtex_splicing_v8_eur_Muscle_Skeletal_signif.db'},
83+
'Nerve_Tibial': {'tissue_id': 'UBERON_0001323', 'db':'gtex_splicing_v8_eur_Nerve_Tibial_signif.db'},
84+
'Ovary': {'tissue_id': 'UBERON_0000992', 'db':'gtex_splicing_v8_eur_Ovary_signif.db'},
85+
'Pancreas': {'tissue_id': 'UBERON_0001150', 'db':'gtex_splicing_v8_eur_Pancreas_signif.db'},
86+
'Pituitary': {'tissue_id': 'UBERON_0000007', 'db':'gtex_splicing_v8_eur_Pituitary_signif.db'},
87+
'Prostate': {'tissue_id': 'UBERON_0002367', 'db':'gtex_splicing_v8_eur_Prostate_signif.db'},
88+
'Skin_Not_Sun_Exposed_Suprapubic': {'tissue_id': 'UBERON_0036149', 'db':'gtex_splicing_v8_eur_Skin_Not_Sun_Exposed_Suprapubic_signif.db'},
89+
'Skin_Sun_Exposed_Lower_leg': {'tissue_id': 'UBERON_0004264', 'db':'gtex_splicing_v8_eur_Skin_Sun_Exposed_Lower_leg_signif.db'},
90+
'Small_Intestine_Terminal_Ileum': {'tissue_id': 'UBERON_0001211', 'db':'gtex_splicing_v8_eur_Small_Intestine_Terminal_Ileum_signif.db'},
91+
'Spleen': {'tissue_id': 'UBERON_0002106', 'db':'gtex_splicing_v8_eur_Spleen_signif.db'},
92+
'Stomach': {'tissue_id': 'UBERON_0000945', 'db':'gtex_splicing_v8_eur_Stomach_signif.db'},
93+
'Testis': {'tissue_id': 'UBERON_0000473', 'db':'gtex_splicing_v8_eur_Testis_signif.db'},
94+
'Thyroid': {'tissue_id': 'UBERON_0002046', 'db':'gtex_splicing_v8_eur_Thyroid_signif.db'},
95+
'Uterus': {'tissue_id': 'UBERON_0000995', 'db':'gtex_splicing_v8_eur_Uterus_signif.db'},
96+
'Vagina': {'tissue_id': 'UBERON_0000996', 'db':'gtex_splicing_v8_eur_Vagina_signif.db'},
97+
'Whole_Blood': {'tissue_id': 'UBERON_0013756', 'db':'gtex_splicing_v8_eur_Whole_Blood_signif.db'}
8898
}
8999

90100

91-
def get_dataset(pmid:int, tissue_id:str) -> Dataset:
101+
def get_dataset(pmid:int, tissue_id:str, name:str) -> Dataset:
92102
''' Fetch the corresponding Dataset from the database '''
93103
try:
94-
dataset = Dataset.objects.get(publication__pmid=pmid, tissue__id=tissue_id)
104+
dataset = Dataset.objects.get(publication__pmid=pmid, tissue__id=tissue_id, name__icontains=name)
95105
return dataset
96106
except Dataset.DoesNotExist:
97107
print(f"Can't find the dataset with PMID:{pmid} and Tissue ID '{tissue_id}'")
98108
exit(0)
99109

100110

111+
def get_score_molecular_trait(score:Score) -> str:
112+
''' Extract the molecular trait ID associated with the score '''
113+
# intron_1_999787_999866_Vagina_GTExV8
114+
score_name = score.name
115+
if score_name.startswith('intron_'):
116+
intron = re.match(r'^(intron_\w+_\d+_\d+)',score_name)
117+
return intron.group(1)
118+
else:
119+
return score.trait_reported_id
120+
121+
101122
def get_genetic_score_data(sqlite_cursor:sqlite3.Cursor,reported_trait_id:str) -> sqlite3.Cursor:
102123
''' Build and execute the SQL query to fetch the genetic score variant information from the SQLite database. '''
124+
# print(f"SELECT rsid, ref_allele, eff_allele, weight FROM weights WHERE gene='{reported_trait_id}';")
103125
qcursor = sqlite_cursor.execute(f"SELECT rsid, ref_allele, eff_allele, weight FROM weights WHERE gene='{reported_trait_id}';")
104126
return qcursor
105127

@@ -201,7 +223,7 @@ def run():
201223

202224
# 2 - Fetch scores and dataset
203225
print(" - Get dataset")
204-
dataset = get_dataset(pmid,datasets_list[dataset_label]['tissue_id'])
226+
dataset = get_dataset(pmid,datasets_list[dataset_label]['tissue_id'],ds_name)
205227

206228
sqlite_file = dbs_location+datasets_list[dataset_label]['db']
207229
sqlite_connection = sqlite3.connect(sqlite_file)
@@ -216,7 +238,8 @@ def run():
216238
for score in scores:
217239
score_id = score.id
218240
genetic_scores[score_id] = []
219-
score_trait = score.trait_reported_id
241+
# score_trait = score.trait_reported_id
242+
score_trait = get_score_molecular_trait(score)
220243
scoring_file_name = f'{dataset_dir}/{score_id}.txt'
221244
# Scoring file
222245
file = open(scoring_file_name,'w')

0 commit comments

Comments
 (0)