Merge pull request #75 from ens-lgil/main

Add new files_ids + updated export scripts

Author: ens-lgil

exports/config_template.py

@@ -10,7 +10,7 @@
     'scoring_files_dir': '<path_to_scoring_files_dataset_directory>', # Uncompressed dataset directory 
     'method_name': '<dataset_method_name>',  # Only needed for GTEx exports
     'platform_name': '<platform_name>',      # Only needed for GTEx exports
-    'use_opgs_id_as_gene': '<True_or_False>' # Use the OmicsPred ID in the 'gene' column or not
+    'use_different_id_as_gene': '<opgs_id_OR_name>' # Use the OmicsPred ID, the score name in the 'gene' column or not. None if the key is missing
 }
 
 

exports/exports.py

@@ -27,6 +27,8 @@
             {'name': 'file_url_scoring_files_pgsc_calc', 'label': 'Scoring files (pgsc_calc compatible)', 'skip_auto_import': True},
             {'name': 'file_url_scoring_files_hm_38', 'label': 'Harmonised scoring files (mapped to GRCh38)', 'skip_auto_import': True},
             {'name': 'file_url_scoring_files', 'label': 'Scoring files', 'skip_auto_import': True},
+            {'name': 'file_url_predictdb', 'label': 'PredictDB', 'skip_auto_import': True},
+            {'name': 'file_url_covariance', 'label': 'Covariance', 'skip_auto_import': True},
             {'name': 'file_url_validation_results', 'label': 'Validation data file', 'skip_auto_import': True},
             {'name': 'file_url_score_variant_info', 'label': 'Score variant info file', 'skip_auto_import': True},
             {'name': 'file_url_gwas_sumstats', 'label': 'GWAS summary stats files', 'skip_auto_import': True}

exports/scripts/generate_sqlite_files.py

@@ -11,10 +11,12 @@
 
 gtex_prefix_study = 'GTExV8'
 
-if 'use_opgs_id_as_gene' in sqlite_default_values.keys():
-    use_opgs_id_as_gene = sqlite_default_values['use_opgs_id_as_gene']
+if 'use_different_id_as_gene' in sqlite_default_values.keys():
+    use_different_id_as_gene = sqlite_default_values['use_different_id_as_gene']
 else:
-    use_opgs_id_as_gene = True
+    use_different_id_as_gene = None
+
+
 
 # default_values = {
 #     'opp_id': 'OPP000003',
@@ -105,6 +107,6 @@ def run(*args):
     # Fetch dataset(s)
     datasets = Dataset.objects.filter(publication__id=opp_id).order_by('id')
     # Create SqliteExport object
-    sqlite_export = SqliteExport(opp_id,output_sqlite_dir,scoring_files_dir,datasets,use_opgs_id_as_gene)
+    sqlite_export = SqliteExport(opp_id,output_sqlite_dir,scoring_files_dir,datasets,use_different_id_as_gene)
     # Generate SQLite export file(s)
     sqlite_export.generate_sqlite_files()

exports/scripts/update_shared_links.py

@@ -0,0 +1,76 @@
+import requests
+from omicspred.models import Dataset
+
+
+def rest_api(folder_id:int, dev_token:str, offset:int=0) -> dict:
+    rest_url = f'https://api.box.com/2.0/folders/{folder_id}/items?fields=name,shared_link'
+    if offset:
+        rest_url += f'&offset={offset}'
+    print(f'- URL: {rest_url}')
+    response = requests.get(rest_url, headers={"Content-Type" : "application/json", 'Authorization': f'Bearer {dev_token}'})
+    response_json = response.json()
+    if response_json:
+        return response_json
+    else:
+        print(response)
+        return None
+
+
+def run(*args):
+    # Update shared links (score.files_ids) using the Box REST API
+
+    # Run example: python manage.py updated_shared_links --script-args 365934779636 metadata 123456 
+    
+    # Generate dev token: 
+    # 1 - Go to https://app.box.com/developers/console
+    # 2 - Click on App "OmicsPred - stats"
+    # 3 - Click on "Configuration" tab
+    # 4 - Click on the "Generate Developer Token" button
+    # 5 - Copy token and add it as parameter to this script
+    if args:
+        folder_id = args[0]
+        file_type = args[1]
+        dev_token = args[2]
+
+    if not folder_id or not file_type or not dev_token:
+        print("- Missing parameter(s): folder_id, file_type or dev_token")
+        exit(1)
+    offset = 0
+    limit = 100
+    total_count = 0
+
+    count_shared_links = 0
+    count_total_files = 0
+    # While loop because of pagination (limit 100)
+    while offset < total_count or total_count == 0 :
+        result = rest_api(folder_id, dev_token, offset)
+        offset += limit
+        total_count = result['total_count']
+        count_total_files = total_count
+        for entry in result['entries']:
+            name = entry['name']
+            # print(f"> Entry: {name}:\n{entry}")
+            # File
+            if entry['type'] == 'file':
+                if 'shared_link' in entry.keys():
+                    # print(entry['shared_link'])
+                    url = entry['shared_link']['url']
+                    dataset_id = name.split('_')[0]
+                    file_id = url.replace('https://app.box.com/s/','')
+                    print(f"- {dataset_id}: {file_id}")
+                    count_shared_links += 1
+                    if dataset_id:
+                        try:
+                            dataset = Dataset.objects.get(id=dataset_id)
+                            files_ids = dataset.files_ids
+                            if file_type not in files_ids.keys():
+                                files_ids[file_type] = file_id
+                                print(f"  > {files_ids}")
+                                dataset.files_ids = files_ids
+                                dataset.save()
+                        except Dataset.DoesNotExist:
+                            print(f"!!!!! Dataset {dataset_id} doesn't exist in the DB")
+                else:
+                    print(f"Shared link missing for {name}!")
+
+    print(f"\n>> Shared links: {count_shared_links}/{count_total_files}")

exports/sqlite_export.py

@@ -73,32 +73,45 @@ class SqliteExport:
     }
 
 
-    def __init__(self, opp_id:str, sqlite_dir:str, scoring_files_dir:str, datasets: list, use_opgs_id_as_gene:bool, insert_block_max:int=100):
+    def __init__(self, opp_id:str, sqlite_dir:str, scoring_files_dir:str, datasets: list, use_different_id_as_gene:bool, insert_block_max:int=100):
         self.opp_id = opp_id
         self.sqlite_dir = sqlite_dir
         self.scoring_files_dir = scoring_files_dir
         self.datasets = datasets # List of Dataset models
-        self.use_opgs_id_as_gene = use_opgs_id_as_gene
+        self.use_different_id_as_gene = use_different_id_as_gene
         self.insert_block_max = insert_block_max
 
 
     def read_scoring_file(self, dataset_dir:str, score_id:str, dataset_name:str, reported_trait_id:str, molecular_traits:dict, genome_build:str) -> list:
         ''' Read content of the scoring file '''
+
+        data = []
+        # Fetch correct directory (usually composed of the dataset ID + suffix)
+        if not os.path.isdir(f'{self.scoring_files_dir}/{dataset_dir}'):
+            for ds_dir in os.listdir(self.scoring_files_dir):
+                if ds_dir.startswith(dataset_dir) and os.path.isdir(f'{self.scoring_files_dir}/{ds_dir}'):
+                    dataset_dir = ds_dir
+                    break
+        if not os.path.isdir(f'{self.scoring_files_dir}/{dataset_dir}'):
+            print(f">> Can't find the directory {self.scoring_files_dir}/{dataset_dir}")
+            exit()
+
+        # Initiate file path(s)
         scoring_file = f'{self.scoring_files_dir}/{dataset_dir}/{score_id}.txt'
         scoring_file_gz = f'{self.scoring_files_dir}/{dataset_dir}/{score_id}.txt.gz'
-        data = []
+
         if not os.path.isfile(scoring_file):
             scoring_file = f'{self.scoring_files_dir}/{dataset_dir}/{score_id}_model.txt'
 
-        # Text file
+        # Case of a text file
         if os.path.isfile(scoring_file):
             with open(scoring_file, newline='') as sc_file:
                 # Ignore lines starting with '#'
                 reader = csv.DictReader(filter(lambda row: row[0]!='#', sc_file), delimiter='\t')
                 for row in reader:
                     variant_info = self.get_variant_info(row,score_id,dataset_name,reported_trait_id,molecular_traits,genome_build)
                     data.append(variant_info)
-        # Zipped file
+        # Case of a zipped file
         elif os.path.isfile(scoring_file_gz):
             with gzip.open(scoring_file_gz, mode="rt") as sc_file:
                 # Ignore lines starting with '#'
@@ -113,18 +126,24 @@ def read_scoring_file(self, dataset_dir:str, score_id:str, dataset_name:str, rep
 
     def get_variant_info(self, row:dict, score_id:str, dataset_name:str, reported_trait_id:str, molecular_traits:dict, genome_build:str) -> tuple:
         ''' Fetch variant information of a row in the scoring file '''
-        rsid = row['rsID'] if 'rsID' in row.keys() else row['rsid']
+        rsid = None
+        if 'rsID' in row.keys():
+            rsid = row['rsID']
+        elif 'rsid' in row.keys():
+            rsid = row['rsid']
         varid = None
         if 'chr_name' in row.keys() and 'chr_position' in row.keys():
             # 1_156000340_A_G_b38
             to_replace = 'GRCh' if genome_build.startswith('GRCh') else 'hg'
             build = 'b'+genome_build.replace(to_replace,'')
             varid = f"{row['chr_name']}_{row['chr_position']}_{row['other_allele']}_{row['effect_allele']}_{build}"
+            if not rsid:
+                rsid = f'chr{row['chr_name']}:{row['chr_position']}'
         elif 'variant_description' in row.keys():
             if row['variant_description'].startswith('variant_id=chr'):
                 varid = row['variant_description'].split('=')[1]
         gene_id = molecular_traits['gene_id']
-        if self.gtex_prefix_study in dataset_name:
+        if (self.gtex_prefix_study in dataset_name) or self.use_different_id_as_gene == 'name':
             variant_info_list = [
                 score_id,             # omicspred_id
                 reported_trait_id,    # gene
@@ -148,7 +167,7 @@ def get_variant_info(self, row:dict, score_id:str, dataset_name:str, reported_tr
 
 
     def create_table_extra(self, cur:sqlite3.Cursor, is_proteomics:bool) -> list:
-        if self.use_opgs_id_as_gene:
+        if self.use_different_id_as_gene == 'opgs_id':
             cols_extra = ['gene', 'gene_id', 'genename', 'gene_type']
         else:
             cols_extra = ['omicspred_id', 'gene', 'genename', 'gene_type']
@@ -161,7 +180,7 @@ def create_table_extra(self, cur:sqlite3.Cursor, is_proteomics:bool) -> list:
 
 
     def create_table_weights(self, cur:sqlite3.Cursor, is_proteomics:bool) -> None:
-        if self.use_opgs_id_as_gene:
+        if self.use_different_id_as_gene == 'opgs_id':
             cols_weights = ['gene', 'gene_id']
         else:
             cols_weights = ['omicspred_id', 'gene']
@@ -214,20 +233,28 @@ def get_score_data(self, dataset_name:str, score:Score, gene:Gene, reported_trai
                 'gene_type': gene.biotype
             }
         else:
-            if self.use_opgs_id_as_gene:
-                score_data = {
-                    'gene': score_id,
-                    'gene_id': gene_id,
-                    'genename': gene.name,
-                    'gene_type': gene.biotype
-                }
-            else:
-                score_data = {
-                    'omicspred_id': score_id,
-                    'gene': gene_id,
-                    'genename': gene.name,
-                    'gene_type': gene.biotype
-                }
+            match self.use_different_id_as_gene:
+                case 'opgs_id':
+                    score_data = {
+                        'gene': score_id,
+                        'gene_id': gene_id,
+                        'genename': gene.name,
+                        'gene_type': gene.biotype
+                    }
+                case 'name':
+                    score_data = {
+                        'omicspred_id': score_id,
+                        'gene': reported_trait_id,
+                        'genename': gene.name,
+                        'gene_type': gene.biotype
+                    }
+                case _:
+                    score_data = {
+                        'omicspred_id': score_id,
+                        'gene': gene_id,
+                        'genename': gene.name,
+                        'gene_type': gene.biotype
+                    }
         if is_proteomics:
             proteins = score.proteins.all()
             protein = proteins[0]
@@ -243,7 +270,7 @@ def get_score_data(self, dataset_name:str, score:Score, gene:Gene, reported_trai
         return score_data
 
 
-    def generate_sqlite_files(self):
+    def generate_sqlite_files(self,skip_zip:bool=False):
         ds_count = 1
         ds_total_count = len(self.datasets)
         for dataset in self.datasets:
@@ -310,6 +337,8 @@ def generate_sqlite_files(self):
                 if self.gtex_prefix_study in dataset_name and score.name.startswith('intron_'):
                     intron_name = score.name.split('_')
                     reported_trait_id = f'{intron_name[0]}_{intron_name[1]}_{intron_name[2]}_{intron_name[3]}'
+                elif self.use_different_id_as_gene == 'name':
+                    reported_trait_id = score.name
                 # Get performance metrics (Training)
                 metrics = self.get_metrics(score)
 
@@ -354,11 +383,12 @@ def generate_sqlite_files(self):
             con.commit()
 
             # Gzip the SQLite file
-            with open(sqlite_filepath, 'rb') as f_in:
-                with gzip.open(f'{sqlite_filepath}.gz', 'wb') as f_out:
-                    shutil.copyfileobj(f_in, f_out)
-            os.remove(sqlite_filepath)
-            print(f"\t-> File zipped")
+            if not skip_zip:
+                with open(sqlite_filepath, 'rb') as f_in:
+                    with gzip.open(f'{sqlite_filepath}.gz', 'wb') as f_out:
+                        shutil.copyfileobj(f_in, f_out)
+                os.remove(sqlite_filepath)
+                print(f"\t-> File zipped")
 
 
             ## FOR test - begin ##

exports/tests/config.py

@@ -8,5 +8,5 @@
     'opp_id': 'OPP000001',
     'sqlite_dir': current_dir+'/exports/tests/output/',
     'scoring_files_dir': current_dir+'/exports/tests/data/scoring_files',
-    'use_opgs_id_as_gene': True
+    'use_different_id_as_gene': 'opgs_id'
 }

exports/tests/test_sqlite_export.py

@@ -18,7 +18,7 @@ class ExportSQLiteTest(TestCase):
     dataset_label = 'DS1'
     current_dir = os.getcwd()
     scoring_files_dir = sqlite_default_values['scoring_files_dir']
-    use_opgs_id_as_gene = sqlite_default_values['use_opgs_id_as_gene']
+    use_different_id_as_gene = sqlite_default_values['use_different_id_as_gene']
 
     sqlite_filename = f'{opd_id}_{dataset_label}.db'
 
@@ -71,8 +71,8 @@ def get_genome_build_data(self,cursor:sqlite3.Cursor):
     def test_sqlite_export(self):
         print("# Test SQLite export")
         datasets = Dataset.objects.filter(id=self.opd_id)
-        sqlite_export = SqliteExport(self.opp_id,self.output_sqlite_dir,self.scoring_files_dir,datasets,self.use_opgs_id_as_gene)
-        sqlite_export.generate_sqlite_files()
+        sqlite_export = SqliteExport(self.opp_id,self.output_sqlite_dir,self.scoring_files_dir,datasets,self.use_different_id_as_gene)
+        sqlite_export.generate_sqlite_files(True)
 
         sqlite_ref_connection = sqlite3.connect(self.ref_sqlite_filepath)
         sqlite_ref_cursor = sqlite_ref_connection.cursor()

rest_api/static/rest_api/openapi/openapi-schema.yml

@@ -298,6 +298,22 @@ components:
               type: string
               description: "Scoring files"
               example: "https://app.box.com/shared/static/z86fg93jg5gwdmmu4xn6u287mre2g5o7"
+            scoring_files_pgsc_calc:
+              type: string
+              description: "Scoring files - pgsc_calc compatible"
+              example: "https://app.box.com/shared/static/7qgaa2ci00x9kggyd48qodxd1ffo3ol7"
+            metadata:
+              type: string
+              description: "Metadata Excel spreadsheet"
+              example: "https://app.box.com/shared/static/anwpkxcpaixbfj67vd6odzqbsb7rk8a8"
+            predictdb:
+              type: string
+              description: "PredictDB prediction weigths - SQLite database"
+              example: "https://app.box.com/shared/static/unjn1bmt2xhwkd33pp0wczldfzrkevu2"
+            covariance:
+              type: string
+              description: "Covariance file, to be used alongside the PredictDB database in MetaXcan tools"
+              example: "https://app.box.com/shared/static/24avhdwb2gfo083oz3r6j8w6qqburbp2"
             score_variant_info:
               type: string
               description: "Variant information file"
@@ -306,10 +322,6 @@ components:
               type: string
               description: "PGS performance metrics"
               example: "https://app.box.com/shared/static/7j7233otah0yxl4rypqx44wzsj7xywxp"
-            scoring_files_pgsc_calc:
-              type: string
-              description: "Scoring files - pgsc_calc compatible"
-              example: "https://app.box.com/shared/static/7qgaa2ci00x9kggyd48qodxd1ffo3ol7"
 
     Dataset_extended:
       description: "Based on the **Dataset** schema, with the addition of the '**publication**'."