Changes related to the reviewers comments

.github/workflows/django_ci.yml

@@ -39,7 +39,7 @@ jobs:
 
     services:
       postgres:
-        image: postgres:15
+        image: postgres:18
         # Provide the user/password/db for postgres
         env:
           POSTGRES_USER: postgres
@@ -58,7 +58,7 @@ jobs:
     strategy:
       max-parallel: 2
       matrix:
-        python-version: [3.13]
+        python-version: [3.14]
 
     steps:
     - uses: actions/checkout@v4

config/settings.py

@@ -267,7 +267,7 @@
 #  REST API Settings  #
 #---------------------#
 
-REST_API_VERSION = '1.2.0'
+REST_API_VERSION = '1.2.1'
 
 #REST_SAFELIST_IPS = [
 #    '127.0.0.1'

exports/config_template.py

@@ -7,8 +7,8 @@
 # 'num__in': ['56','105','154','203']
 
 # Metadata exports
-metadata_exports_publication_id = '<publication_id, e.g. 1>'
 metadata_exports_dir = '<path_to_directory>'
+sqlite_exports_dir = '<path_to_sqlite_directory>' # Uncompressed databases
 
 # PheWAS exports
 phewas_exports_dir = '<path_to_directory>'

exports/exports.py

@@ -17,6 +17,7 @@
             {'name': 'name', 'label': 'Dataset Name'},
             {'name': 'omics_type', 'label': 'Omics Type' },
             {'name': 'method_name', 'label': 'Development Method'},
+            {'name': 'training_window', 'label': 'Training Window'},
             {'name': 'scores_count', 'label': 'Scores Count'},
             {'name': 'phewas_count', 'label': 'PheWAS Count'},
             {'name': 'platform__name', 'label': 'Platform Name'},
@@ -29,7 +30,6 @@
             {'name': 'file_url_scoring_files_hm_38', 'label': 'Harmonised scoring files (mapped to GRCh38)', 'skip_auto_import': True},
             {'name': 'file_url_scoring_files', 'label': 'Scoring files', 'skip_auto_import': True},
             {'name': 'file_url_predictdb', 'label': 'PredictDB', 'skip_auto_import': True},
-            {'name': 'file_url_covariance', 'label': 'Covariance', 'skip_auto_import': True},
             {'name': 'file_url_phewas', 'label': 'PheWAS', 'skip_auto_import': True},
             {'name': 'file_url_phewas_full', 'label': 'PheWAS Full', 'skip_auto_import': True},
             {'name': 'file_url_validation_results', 'label': 'Validation data file', 'skip_auto_import': True},
@@ -71,7 +71,9 @@
             {'name': 'sample__sample_age', 'label': 'Mean Sample Age'},
             {'name': 'sample__sample_age_sd', 'label': 'Standard Deviation of Age'},
             {'name': 'sample__ancestry_broad', 'label': 'Broad Ancestry Category'},
+            {'name': 'sample__ancestry_assignment', 'label': 'Ancestry Assignment Method'},
             {'name': 'cohorts', 'label': 'Cohort(s)', 'skip_auto_import': True},
+            {'name': 'sample__cohorts_additional', 'label': 'Cohort(s) additional information'},
             {'name': 'metrics_r2', 'label': 'R2', 'skip_auto_import': True},
             {'name': 'metrics_r2_pval', 'label': 'R2 - p-value', 'skip_auto_import': True},
             {'name': 'metrics_rho', 'label': 'Rho', 'skip_auto_import': True},

exports/metadata_build_export.py

@@ -13,7 +13,7 @@ class MetadataExport:
     def __init__(self, exports_dir:str, sqlite_dir:str, dataset:Dataset):
         self.dataset = dataset
         self.dataset_id = dataset.id
-        self.sqlite_dir = sqlite_dir
+        self.sqlite_dir = f'{sqlite_dir}/{self.dataset_id}'
         self.data = {
             'dataset': self.get_data_attr(self.dataset,'Dataset'),
             'publication': self.get_data_attr(self.dataset.publication,'Publication'),
@@ -26,15 +26,16 @@ def __init__(self, exports_dir:str, sqlite_dir:str, dataset:Dataset):
 
         # Find corresponding SQLite file
         self.sqlite_file = None
-        for s_file in os.listdir(sqlite_dir):
+        for s_file in os.listdir(self.sqlite_dir):
             if s_file.startswith(dataset.id) and s_file.endswith('.db'):
                 self.sqlite_file = (s_file)
                 break
         if not self.sqlite_file:
-            print(f"ERROR: Can't find a SQLite file for the dataset {dataset.id} in {sqlite_dir}")
+            print(f"ERROR: Can't find a SQLite file for the dataset {self.dataset_id} in {self.sqlite_dir}")
             exit()
 
-        self.filename = f'{exports_dir}/{self.dataset_id}_metadata.xlsx'
+        filename = self.sqlite_file.replace('.db','_metadata.xlsx')
+        self.filepath = f'{exports_dir}/{filename}'
 
 
     def get_data_attr(self, model:object, model_type:str)-> dict:
@@ -99,6 +100,9 @@ def build_performance_metadata(self, score:Score):
             # Update eval_type using the long name
             sample_perf_data['eval_type'] = perf.get_eval_type_display()
 
+            # Update the Sample ancestry_assignment using the long name
+            sample_perf_data['sample__ancestry_assignment'] = perf.sample.get_ancestry_assignment_display()
+
             # Cohorts
             cohorts_list = []
             for cohort in perf.sample.cohorts.all():
@@ -206,6 +210,6 @@ def generate_metadata(self):
             self.build_performance_metadata(score)
 
         # Create export
-        op_export = OPExport(self.filename, self.data)
+        op_export = OPExport(self.filepath, self.data)
         op_export.generate_sheets()
         op_export.save()

exports/scripts/generate_metadata_exports.py

@@ -2,8 +2,7 @@
 from omicspred.models import Dataset
 from exports.metadata_build_export import MetadataExport
 from exports.datasets import DatasetsSelection
-from exports.config import metadata_exports_dir, metadata_exports_publication_id, sqlite_exports_dir
-from django.db.models import Q
+from exports.config import metadata_exports_dir, sqlite_exports_dir
 
 
 def run():

exports/tests/config.py

@@ -1,7 +1,9 @@
 import os
 
+dataset_selection = { 'publication_id': 1 }
+
 current_dir = os.getcwd()
-metadata_exports_publication_id = 1
+sqlite_exports_dir = current_dir+'/exports/tests/datas'
 metadata_exports_dir = current_dir+'/exports/tests/output'
 phewas_exports_dir = current_dir+'/exports/tests/output'
 sqlite_exports_dir = current_dir+'/exports/tests/data/'

exports/tests/test_sqlite_export.py

@@ -23,8 +23,8 @@ class ExportSQLiteTest(TestCase):
     sqlite_filename = f'{opd_id}_{dataset_label}.db'
 
     # SQLite ref
-    ref_sqlite_dir = current_dir+'/exports/tests/data/'
-    ref_sqlite_filepath = f'{ref_sqlite_dir}{sqlite_filename}'
+    ref_sqlite_dir = current_dir+'/exports/tests/data/'+opd_id
+    ref_sqlite_filepath = f'{ref_sqlite_dir}/{sqlite_filename}'
     # SQLite Output
     output_sqlite_dir = sqlite_default_values['sqlite_dir']
     output_sqlite_filepath = f'{output_sqlite_dir}{sqlite_filename}'

exports/tests/tests_metadata_export.py

@@ -1,9 +1,10 @@
 import os
 import pandas as pd
 from django.test import TestCase
+from django.db.models import Q
 from omicspred.models import *
 from exports.metadata_build_export import MetadataExport
-from exports.tests.config import metadata_exports_dir,metadata_exports_publication_id, sqlite_exports_dir
+from exports.tests.config import metadata_exports_dir, sqlite_exports_dir, dataset_selection
 
 
 class ExportMetadataTest(TestCase):
@@ -12,13 +13,13 @@ class ExportMetadataTest(TestCase):
     # Load data in DB - from the rest_api/fixtures/ directory
     fixtures = ['db_test']
     databases = {'default'}
-    filename = 'OPD000001_metadata.xlsx'
+    filename = 'OPD000001_DS1_metadata.xlsx'
 
     spreadsheets = ['Publication','Dataset','Scores','Performances','Cohorts']
 
     current_dir = os.getcwd()
     output_export_dir = metadata_exports_dir
-    data_export_dir = current_dir+'/exports/tests/data'
+    data_export_dir = current_dir+'/exports/tests/data/OPD000001'
     output_filepath = f'{output_export_dir}/{filename}'
     # Remove output metadata file before creating it again
     if os.path.isdir(output_export_dir):
@@ -36,7 +37,8 @@ class ExportMetadataTest(TestCase):
         except OSError:
             print (f'Creation of the directory {output_export_dir} failed')
             exit()
-
+
+
     def check_file(self):
         ''' Check file exists and is not empty '''
         print(f' - Check file {self.filename}')
@@ -58,14 +60,20 @@ def check_spreadsheets(self):
             df_test = pd.read_excel(self.output_filepath, sheet_name=sp_name, index_col=0)
             df_ref = pd.read_excel(ref_filepath, sheet_name=sp_name, index_col=0)
             # Columns
-            self.assertEqual(df_test.columns.all(),df_ref.columns.all())
+            self.assertEqual(len(df_test.columns),len(df_ref.columns))
             # Rows
             self.assertEqual(len(df_test.index), len(df_ref.index))
 
 
     def test_metadata_export(self):
         print("# Test Metadata export")
-        datasets = Dataset.objects.filter(publication_id=metadata_exports_publication_id).order_by('num')
+
+        # Mimic DatasetsSelection
+        col = list(dataset_selection.keys())[0]
+        value = dataset_selection[col]
+        param = Q(**{f'{col}':value})
+
+        datasets = Dataset.objects.filter(param).order_by('num')
         dataset = datasets[0]
         metadata2export = MetadataExport(self.output_export_dir,sqlite_exports_dir, dataset)
         metadata2export.generate_metadata()

imports/omicspred/models/platform.py

@@ -8,7 +8,7 @@
 
 class PlatformMasterData(GenericData):
 
-    def __init__(self,name,type,full_name=None,technic=None):
+    def __init__(self,name,type,full_name=None,technique=None):
         GenericData.__init__(self)
         self.name = name
         self.data = {
@@ -17,8 +17,8 @@ def __init__(self,name,type,full_name=None,technic=None):
         }
         if full_name:
             self.data['full_name'] = full_name
-        if technic:
-            self.data['technic'] = technic
+        if technique:
+            self.data['technique'] = technique
 
 
     def check_model_exist(self):

imports/omicspred/parsers/data_content.py

@@ -16,7 +16,7 @@
         'method_name': method_name,
         'full_name': 'Olink',
         'version': 'Explore',
-        'technic': 'antibody-based proximity extension assay for proteins',
+        'technique': 'antibody-based proximity extension assay for proteins',
         'dataset_name': 'UKB European',
         'internal_cohort': 'UKB',
         'internal_label': internal_label,
@@ -43,7 +43,7 @@
         'method_name': method_name,
         'full_name': 'Olink',
         'version': 'Explore',
-        'technic': 'antibody-based proximity extension assay for proteins',
+        'technique': 'antibody-based proximity extension assay for proteins',
         'dataset_name': 'UKB Multi-ancestry',
         'internal_cohort': 'UKB',
         'internal_label': internal_label,

omicspred/migrations/0001_initial.py

@@ -1,4 +1,4 @@
-# Generated by Django 6.0.5 on 2026-05-12 10:40
+# Generated by Django 6.0.6 on 2026-07-03 13:41
 
 import django.contrib.postgres.fields.ranges
 import django.core.validators
@@ -34,6 +34,7 @@ class Migration(migrations.Migration):
                 ('omics_count', models.IntegerField(verbose_name='Omics Entities count')),
                 ('omics_type', models.CharField(max_length=50, verbose_name='Omics type')),
                 ('method_name', models.TextField(verbose_name='Score Development Method')),
+                ('training_window', models.CharField(choices=[('', ''), ('genome-wide', 'Genome-wide'), ('cis-only', 'Cis-only')], db_index=True, default='', max_length=25)),
                 ('scores_count', models.IntegerField(verbose_name='Associated Scores count')),
                 ('phewas_count', models.IntegerField(default=0, verbose_name='Associated PheWAS data count')),
                 ('files_ids', models.JSONField(default=dict, verbose_name='Files IDs on Box')),
@@ -43,6 +44,15 @@ class Migration(migrations.Migration):
                 'get_latest_by': 'num',
             },
         ),
+        migrations.CreateModel(
+            name='ExternalSource',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=50, verbose_name='External Source Name')),
+                ('version', models.CharField(max_length=50, null=True, verbose_name='External Source Version')),
+                ('url', models.CharField(max_length=100, verbose_name='External Source URL')),
+            ],
+        ),
         migrations.CreateModel(
             name='Pathway',
             fields=[
@@ -89,7 +99,7 @@ class Migration(migrations.Migration):
                 ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
                 ('name', models.CharField(max_length=100, verbose_name='Platform name')),
                 ('full_name', models.CharField(max_length=100, verbose_name='Platform full name')),
-                ('technic', models.CharField(max_length=100, verbose_name='Platform technic')),
+                ('technique', models.CharField(max_length=100, verbose_name='Platform technique')),
                 ('type', models.CharField(max_length=100, verbose_name='Platform type')),
             ],
         ),
@@ -280,6 +290,7 @@ class Migration(migrations.Migration):
                 ('ancestry_free', models.TextField(null=True, verbose_name='Ancestry (e.g. French, Chinese)')),
                 ('ancestry_country', models.TextField(null=True, verbose_name='Country of Recruitment')),
                 ('ancestry_additional', models.TextField(null=True, verbose_name='Additional Ancestry Description')),
+                ('ancestry_assignment', models.CharField(choices=[('SR', 'Self reported'), ('GS', 'Genetic similarity'), ('GSR', 'Genetic similarity to reference panel'), ('AS', 'Author statement - no method reported'), ('INF', 'Curator inferred ancestry label from cohort description (e.g. country of recruitment)'), ('NR', 'No population descriptor or inferrable CoR provided')], default='SR', max_length=4)),
                 ('source_gwas_catalog', models.CharField(max_length=20, null=True, verbose_name='GWAS Catalog Study ID (GCST...)')),
                 ('source_pmid', models.IntegerField(null=True, verbose_name='Source PubMed ID (PMID)')),
                 ('source_doi', models.CharField(max_length=100, null=True, verbose_name='Source DOI')),
@@ -352,7 +363,7 @@ class Migration(migrations.Migration):
                 ('var_gene_exp', models.FloatField(null=True, verbose_name='Variance of the gene expression')),
                 ('variants_number_used', models.IntegerField(null=True, verbose_name='Number of variants used')),
                 ('variants_fraction_found', models.FloatField(null=True, verbose_name='Fraction of variants found')),
-                ('dataset', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='dataset_phenotype', to='omicspred.dataset', verbose_name='Dataset')),
+                ('dataset', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='dataset_phewas', to='omicspred.dataset', verbose_name='Dataset')),
                 ('phenotypes', models.ManyToManyField(related_name='phenotype_scores', to='omicspred.phenotype', verbose_name='Phenotype(s)')),
                 ('publication', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, related_name='score_phewas_publication', to='omicspred.publication', verbose_name='PheWAS Publication')),
                 ('samples', models.ManyToManyField(related_name='sample_scores', to='omicspred.sample', verbose_name='Sample(s)')),

omicspred/models.py

@@ -107,7 +107,7 @@ class PlatformMaster(models.Model):
     """ Class to describe the platform used to get the omics data """
     name = models.CharField('Platform name', max_length=100)
     full_name = models.CharField('Platform full name', max_length=100)
-    technic = models.CharField('Platform technic', max_length=100)
+    technique = models.CharField('Platform technique', max_length=100)
     type = models.CharField('Platform type', max_length=100)
 
     @property
@@ -132,10 +132,7 @@ def scores_count(self):
 class Platform(models.Model):
     """ Class to describe the versioned platform used to get the omics data """
     name = models.CharField('Platform name', max_length=100, db_index=True)
-    # full_name = models.CharField('Platform full name', max_length=100)
     version = models.CharField('Platform version', max_length=50)
-    # technic = models.CharField('Platform technic', max_length=100)
-    # type = models.CharField('Platform type', max_length=100)
     platform_master = models.ForeignKey(PlatformMaster, on_delete=models.CASCADE, related_name='platform_version', verbose_name='Platform')
 
     @property
@@ -185,6 +182,19 @@ class Sample(models.Model):
     ancestry_free = models.TextField('Ancestry (e.g. French, Chinese)', null=True)
     ancestry_country = models.TextField('Country of Recruitment', null=True)
     ancestry_additional = models.TextField('Additional Ancestry Description', null=True)
+    ANCESTRY_ASSIGNMENT_CHOICES = [
+        ('SR', 'Self reported'),
+        ('GS', 'Genetic similarity'),
+        ('GSR', 'Genetic similarity to reference panel'),
+        ('AS', 'Author statement - no method reported'),
+        ('INF', 'Curator inferred ancestry label from cohort description (e.g. country of recruitment)'),
+        ('NR', 'No population descriptor or inferrable CoR provided')
+    ]
+    ancestry_assignment = models.CharField(max_length=4,
+        choices=ANCESTRY_ASSIGNMENT_CHOICES,
+        default='SR',
+        db_index=False
+    )
 
     ## Cohorts/Sources
     source_gwas_catalog = models.CharField('GWAS Catalog Study ID (GCST...)', max_length=20, null=True)
@@ -324,6 +334,17 @@ class Dataset(models.Model):
     omics_count = models.IntegerField('Omics Entities count', null=False)
     omics_type = models.CharField('Omics type', max_length=50)
     method_name = models.TextField('Score Development Method')
+    # Cis-variant, Trans-variants
+    TRAINING_WINDOW_CHOICES = [
+        ('',''),
+        ('genome-wide', 'Genome-wide'),
+        ('cis-only', 'Cis-only')
+    ]
+    training_window = models.CharField(max_length=25,
+        choices=TRAINING_WINDOW_CHOICES,
+        default='',
+        db_index=True
+    )
     tissue = models.ForeignKey(Tissue, on_delete=models.PROTECT, related_name='tissue_dataset', verbose_name='Tissue', null=True) # Tissue trait defining the sampled tissue
     scores_count = models.IntegerField('Associated Scores count', null=False)
     phewas_count = models.IntegerField('Associated PheWAS data count', default=0)
@@ -782,4 +803,11 @@ def values_dict(self):
             'bonferroni': self.bonferroni,
             'effect_size': self.effect_size,
             'var_gene_exp': self.var_gene_exp
-        }
+        }
+
+
+class ExternalSource(models.Model):
+    """ Class to hold ExternalSource values """
+    name = models.CharField('External Source Name', max_length=50)
+    version = models.CharField('External Source Version', max_length=50, null=True)
+    url = models.CharField('External Source URL', max_length=100)