Skip to content

Study Index

gentropy.dataset.study_index.StudyIndex dataclass

Bases: Dataset

Study index dataset.

A study index dataset captures all the metadata for all studies including GWAS and Molecular QTL.

Source code in src/gentropy/dataset/study_index.py
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
@dataclass
class StudyIndex(Dataset):
    """Study index dataset.

    A study index dataset captures all the metadata for all studies including GWAS and Molecular QTL.
    """

    @staticmethod
    def _aggregate_samples_by_ancestry(merged: Column, ancestry: Column) -> Column:
        """Aggregate sample counts by ancestry in a list of struct colmns.

        Args:
            merged (Column): A column representing merged data (list of structs).
            ancestry (Column): The `ancestry` parameter is a column that represents the ancestry of each
                sample. (a struct)

        Returns:
            Column: the modified "merged" column after aggregating the samples by ancestry.
        """
        # Iterating over the list of ancestries and adding the sample size if label matches:
        return f.transform(
            merged,
            lambda a: f.when(
                a.ancestry == ancestry.ancestry,
                f.struct(
                    a.ancestry.alias("ancestry"),
                    (a.sampleSize + ancestry.sampleSize).alias("sampleSize"),
                ),
            ).otherwise(a),
        )

    @staticmethod
    def _map_ancestries_to_ld_population(gwas_ancestry_label: Column) -> Column:
        """Normalise ancestry column from GWAS studies into reference LD panel based on a pre-defined map.

        This function assumes all possible ancestry categories have a corresponding
        LD panel in the LD index. It is very important to have the ancestry labels
        moved to the LD panel map.

        Args:
            gwas_ancestry_label (Column): A struct column with ancestry label like Finnish,
                European, African etc. and the corresponding sample size.

        Returns:
            Column: Struct column with the mapped LD population label and the sample size.
        """
        # Loading ancestry label to LD population label:
        json_dict = json.loads(
            pkg_resources.read_text(
                data, "gwas_population_2_LD_panel_map.json", encoding="utf-8"
            )
        )
        map_expr = f.create_map(*[f.lit(x) for x in chain(*json_dict.items())])

        return f.struct(
            map_expr[gwas_ancestry_label.ancestry].alias("ancestry"),
            gwas_ancestry_label.sampleSize.alias("sampleSize"),
        )

    @classmethod
    def get_schema(cls: type[StudyIndex]) -> StructType:
        """Provide the schema for the StudyIndex dataset.

        Returns:
            StructType: The schema of the StudyIndex dataset.
        """
        return parse_spark_schema("study_index.json")

    @classmethod
    def aggregate_and_map_ancestries(
        cls: type[StudyIndex], discovery_samples: Column
    ) -> Column:
        """Map ancestries to populations in the LD reference and calculate relative sample size.

        Args:
            discovery_samples (Column): A list of struct column. Has an `ancestry` column and a `sampleSize` columns

        Returns:
            Column: A list of struct with mapped LD population and their relative sample size.
        """
        # Map ancestry categories to population labels of the LD index:
        mapped_ancestries = f.transform(
            discovery_samples, cls._map_ancestries_to_ld_population
        )

        # Aggregate sample sizes belonging to the same LD population:
        aggregated_counts = f.aggregate(
            mapped_ancestries,
            f.array_distinct(
                f.transform(
                    mapped_ancestries,
                    lambda x: f.struct(
                        x.ancestry.alias("ancestry"), f.lit(0.0).alias("sampleSize")
                    ),
                )
            ),
            cls._aggregate_samples_by_ancestry,
        )
        # Getting total sample count:
        total_sample_count = f.aggregate(
            aggregated_counts, f.lit(0.0), lambda total, pop: total + pop.sampleSize
        ).alias("sampleSize")

        # Calculating relative sample size for each LD population:
        return f.transform(
            aggregated_counts,
            lambda ld_population: f.struct(
                ld_population.ancestry.alias("ldPopulation"),
                (ld_population.sampleSize / total_sample_count).alias(
                    "relativeSampleSize"
                ),
            ),
        )

    def study_type_lut(self: StudyIndex) -> DataFrame:
        """Return a lookup table of study type.

        Returns:
            DataFrame: A dataframe containing `studyId` and `studyType` columns.
        """
        return self.df.select("studyId", "studyType")

    def is_qtl(self: StudyIndex) -> Column:
        """Return a boolean column with true values for QTL studies.

        Returns:
            Column: True if the study is a QTL study.
        """
        return self.df.studyType.endswith("qtl")

    def is_gwas(self: StudyIndex) -> Column:
        """Return a boolean column with true values for GWAS studies.

        Returns:
            Column: True if the study is a GWAS study.
        """
        return self.df.studyType == "gwas"

    def has_mapped_trait(self: StudyIndex) -> Column:
        """Return a boolean column indicating if a study has mapped disease.

        Returns:
            Column: True if the study has mapped disease.
        """
        return f.size(self.df.traitFromSourceMappedIds) > 0

    def is_quality_flagged(self: StudyIndex) -> Column:
        """Return a boolean column indicating if a study is flagged due to quality issues.

        Returns:
            Column: True if the study is flagged.
        """
        # Testing for the presence of the qualityControls column:
        if "qualityControls" not in self.df.columns:
            return f.lit(False)
        else:
            return f.size(self.df.qualityControls) != 0

    def has_summarystats(self: StudyIndex) -> Column:
        """Return a boolean column indicating if a study has harmonized summary statistics.

        Returns:
            Column: True if the study has harmonized summary statistics.
        """
        return self.df.hasSumstats

    def validate_unique_study_id(self: StudyIndex) -> StudyIndex:
        """Validating the uniqueness of study identifiers and flagging duplicated studies.

        Returns:
            StudyIndex: with flagged duplicated studies.
        """
        validated_df = (
            self.df.withColumn(
                "isDuplicated",
                f.when(
                    f.count("studyType").over(
                        Window.partitionBy("studyId").rowsBetween(
                            Window.unboundedPreceding, Window.unboundedFollowing
                        )
                    )
                    > 1,
                    True,
                ).otherwise(False),
            )
            .withColumn(
                "qualityControls",
                StudyIndex.update_quality_flag(
                    f.col("qualityControls"),
                    f.col("isDuplicated"),
                    StudyQualityCheck.DUPLICATED_STUDY,
                ),
            )
            .drop("isDuplicated")
        )
        return StudyIndex(_df=validated_df, _schema=StudyIndex.get_schema())

    def _normalise_disease(
        self: StudyIndex,
        source_disease_column_name: str,
        disease_column_name: str,
        disease_map: DataFrame,
    ) -> DataFrame:
        """Normalising diseases in the study index.

        Given a reference disease map (containing all potential EFO ids with the corresponding reference disease ids),
        this function maps all EFO ids in the study index to the reference disease ids.

        Args:
            source_disease_column_name (str): The column name of the disease column to validate.
            disease_column_name (str): The resulting disease column name that contains the validated ids.
            disease_map (DataFrame): Reference dataframe with diseases

        Returns:
            DataFrame: where the newly added diseaseIds column will contain the validated EFO identifiers.
        """
        return (
            self.df
            # Only validating studies with diseases:
            .filter(f.size(f.col(source_disease_column_name)) > 0)
            # Explode disease column:
            .select(
                "studyId",
                "studyType",
                f.explode_outer(source_disease_column_name).alias("efo"),
            )
            # Join disease map:
            .join(disease_map, on="efo", how="left")
            .groupBy("studyId")
            .agg(
                f.collect_set(f.col("diseaseId")).alias(disease_column_name),
            )
        )

    def validate_disease(self: StudyIndex, disease_map: DataFrame) -> StudyIndex:
        """Validate diseases in the study index dataset.

        Args:
            disease_map (DataFrame): a dataframe with two columns (efo, diseaseId).

        Returns:
            StudyIndex: where gwas studies are flagged where no valid disease id could be found.
        """
        # Because the disease ids are not mandatory fields of the schema, we skip vaildation if these columns are not present:
        if ("traitFromSourceMappedIds" not in self.df.columns) or (
            "backgroundTraitFromSourceMappedIds" not in self.df.columns
        ):
            return self

        # Disease Column names:
        foreground_disease_column = "diseaseIds"
        background_disease_column = "backgroundDiseaseIds"

        # If diseaseId in schema, we need to drop it:
        drop_columns = [
            column
            for column in self.df.columns
            if column in [foreground_disease_column, background_disease_column]
        ]

        if len(drop_columns) > 0:
            self.df = self.df.drop(*drop_columns)

        # Normalise disease:
        normalised_disease = self._normalise_disease(
            "traitFromSourceMappedIds", foreground_disease_column, disease_map
        )
        normalised_background_disease = self._normalise_disease(
            "backgroundTraitFromSourceMappedIds", background_disease_column, disease_map
        )

        return StudyIndex(
            _df=(
                self.df.join(normalised_disease, on="studyId", how="left")
                .join(normalised_background_disease, on="studyId", how="left")
                # Updating disease columns:
                .withColumn(
                    foreground_disease_column,
                    f.when(
                        f.col(foreground_disease_column).isNull(), f.array()
                    ).otherwise(f.col(foreground_disease_column)),
                )
                .withColumn(
                    background_disease_column,
                    f.when(
                        f.col(background_disease_column).isNull(), f.array()
                    ).otherwise(f.col(background_disease_column)),
                )
                # Flagging gwas studies where no valid disease is avilable:
                .withColumn(
                    "qualityControls",
                    StudyIndex.update_quality_flag(
                        f.col("qualityControls"),
                        # Flagging all gwas studies with no normalised disease:
                        (f.size(f.col(foreground_disease_column)) == 0)
                        & (f.col("studyType") == "gwas"),
                        StudyQualityCheck.UNRESOLVED_DISEASE,
                    ),
                )
            ),
            _schema=StudyIndex.get_schema(),
        )

    def validate_study_type(self: StudyIndex) -> StudyIndex:
        """Validating study type and flag unsupported types.

        Returns:
            StudyIndex: with flagged studies with unsupported type.
        """
        validated_df = (
            self.df
            # Flagging unsupported study types:
            .withColumn(
                "qualityControls",
                StudyIndex.update_quality_flag(
                    f.col("qualityControls"),
                    f.when(
                        (f.col("studyType") == "gwas")
                        | f.col("studyType").endswith("qtl"),
                        False,
                    ).otherwise(True),
                    StudyQualityCheck.UNKNOWN_STUDY_TYPE,
                ),
            )
        )
        return StudyIndex(_df=validated_df, _schema=StudyIndex.get_schema())

    def validate_target(self: StudyIndex, target_index: GeneIndex) -> StudyIndex:
        """Validating gene identifiers in the study index against the provided target index.

        Args:
            target_index (GeneIndex): gene index containing the reference gene identifiers (Ensembl gene identifiers).

        Returns:
            StudyIndex: with flagged studies if geneId could not be validated.
        """
        gene_set = target_index.df.select("geneId", f.lit(True).alias("isIdFound"))

        # As the geneId is not a mandatory field of study index, we return if the column is not there:
        if "geneId" not in self.df.columns:
            return self

        validated_df = (
            self.df.join(gene_set, on="geneId", how="left")
            .withColumn(
                "isIdFound",
                f.when(
                    (f.col("studyType") != "gwas") & f.col("isIdFound").isNull(),
                    f.lit(False),
                ).otherwise(f.lit(True)),
            )
            .withColumn(
                "qualityControls",
                StudyIndex.update_quality_flag(
                    f.col("qualityControls"),
                    ~f.col("isIdFound"),
                    StudyQualityCheck.UNRESOLVED_TARGET,
                ),
            )
            .drop("isIdFound")
        )

        return StudyIndex(_df=validated_df, _schema=StudyIndex.get_schema())

aggregate_and_map_ancestries(discovery_samples: Column) -> Column classmethod

Map ancestries to populations in the LD reference and calculate relative sample size.

Parameters:

Name Type Description Default
discovery_samples Column

A list of struct column. Has an ancestry column and a sampleSize columns

required

Returns:

Name Type Description
Column Column

A list of struct with mapped LD population and their relative sample size.

Source code in src/gentropy/dataset/study_index.py
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
@classmethod
def aggregate_and_map_ancestries(
    cls: type[StudyIndex], discovery_samples: Column
) -> Column:
    """Map ancestries to populations in the LD reference and calculate relative sample size.

    Args:
        discovery_samples (Column): A list of struct column. Has an `ancestry` column and a `sampleSize` columns

    Returns:
        Column: A list of struct with mapped LD population and their relative sample size.
    """
    # Map ancestry categories to population labels of the LD index:
    mapped_ancestries = f.transform(
        discovery_samples, cls._map_ancestries_to_ld_population
    )

    # Aggregate sample sizes belonging to the same LD population:
    aggregated_counts = f.aggregate(
        mapped_ancestries,
        f.array_distinct(
            f.transform(
                mapped_ancestries,
                lambda x: f.struct(
                    x.ancestry.alias("ancestry"), f.lit(0.0).alias("sampleSize")
                ),
            )
        ),
        cls._aggregate_samples_by_ancestry,
    )
    # Getting total sample count:
    total_sample_count = f.aggregate(
        aggregated_counts, f.lit(0.0), lambda total, pop: total + pop.sampleSize
    ).alias("sampleSize")

    # Calculating relative sample size for each LD population:
    return f.transform(
        aggregated_counts,
        lambda ld_population: f.struct(
            ld_population.ancestry.alias("ldPopulation"),
            (ld_population.sampleSize / total_sample_count).alias(
                "relativeSampleSize"
            ),
        ),
    )

get_schema() -> StructType classmethod

Provide the schema for the StudyIndex dataset.

Returns:

Name Type Description
StructType StructType

The schema of the StudyIndex dataset.

Source code in src/gentropy/dataset/study_index.py
103
104
105
106
107
108
109
110
@classmethod
def get_schema(cls: type[StudyIndex]) -> StructType:
    """Provide the schema for the StudyIndex dataset.

    Returns:
        StructType: The schema of the StudyIndex dataset.
    """
    return parse_spark_schema("study_index.json")

has_mapped_trait() -> Column

Return a boolean column indicating if a study has mapped disease.

Returns:

Name Type Description
Column Column

True if the study has mapped disease.

Source code in src/gentropy/dataset/study_index.py
182
183
184
185
186
187
188
def has_mapped_trait(self: StudyIndex) -> Column:
    """Return a boolean column indicating if a study has mapped disease.

    Returns:
        Column: True if the study has mapped disease.
    """
    return f.size(self.df.traitFromSourceMappedIds) > 0

has_summarystats() -> Column

Return a boolean column indicating if a study has harmonized summary statistics.

Returns:

Name Type Description
Column Column

True if the study has harmonized summary statistics.

Source code in src/gentropy/dataset/study_index.py
202
203
204
205
206
207
208
def has_summarystats(self: StudyIndex) -> Column:
    """Return a boolean column indicating if a study has harmonized summary statistics.

    Returns:
        Column: True if the study has harmonized summary statistics.
    """
    return self.df.hasSumstats

is_gwas() -> Column

Return a boolean column with true values for GWAS studies.

Returns:

Name Type Description
Column Column

True if the study is a GWAS study.

Source code in src/gentropy/dataset/study_index.py
174
175
176
177
178
179
180
def is_gwas(self: StudyIndex) -> Column:
    """Return a boolean column with true values for GWAS studies.

    Returns:
        Column: True if the study is a GWAS study.
    """
    return self.df.studyType == "gwas"

is_qtl() -> Column

Return a boolean column with true values for QTL studies.

Returns:

Name Type Description
Column Column

True if the study is a QTL study.

Source code in src/gentropy/dataset/study_index.py
166
167
168
169
170
171
172
def is_qtl(self: StudyIndex) -> Column:
    """Return a boolean column with true values for QTL studies.

    Returns:
        Column: True if the study is a QTL study.
    """
    return self.df.studyType.endswith("qtl")

is_quality_flagged() -> Column

Return a boolean column indicating if a study is flagged due to quality issues.

Returns:

Name Type Description
Column Column

True if the study is flagged.

Source code in src/gentropy/dataset/study_index.py
190
191
192
193
194
195
196
197
198
199
200
def is_quality_flagged(self: StudyIndex) -> Column:
    """Return a boolean column indicating if a study is flagged due to quality issues.

    Returns:
        Column: True if the study is flagged.
    """
    # Testing for the presence of the qualityControls column:
    if "qualityControls" not in self.df.columns:
        return f.lit(False)
    else:
        return f.size(self.df.qualityControls) != 0

study_type_lut() -> DataFrame

Return a lookup table of study type.

Returns:

Name Type Description
DataFrame DataFrame

A dataframe containing studyId and studyType columns.

Source code in src/gentropy/dataset/study_index.py
158
159
160
161
162
163
164
def study_type_lut(self: StudyIndex) -> DataFrame:
    """Return a lookup table of study type.

    Returns:
        DataFrame: A dataframe containing `studyId` and `studyType` columns.
    """
    return self.df.select("studyId", "studyType")

validate_disease(disease_map: DataFrame) -> StudyIndex

Validate diseases in the study index dataset.

Parameters:

Name Type Description Default
disease_map DataFrame

a dataframe with two columns (efo, diseaseId).

required

Returns:

Name Type Description
StudyIndex StudyIndex

where gwas studies are flagged where no valid disease id could be found.

Source code in src/gentropy/dataset/study_index.py
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
def validate_disease(self: StudyIndex, disease_map: DataFrame) -> StudyIndex:
    """Validate diseases in the study index dataset.

    Args:
        disease_map (DataFrame): a dataframe with two columns (efo, diseaseId).

    Returns:
        StudyIndex: where gwas studies are flagged where no valid disease id could be found.
    """
    # Because the disease ids are not mandatory fields of the schema, we skip vaildation if these columns are not present:
    if ("traitFromSourceMappedIds" not in self.df.columns) or (
        "backgroundTraitFromSourceMappedIds" not in self.df.columns
    ):
        return self

    # Disease Column names:
    foreground_disease_column = "diseaseIds"
    background_disease_column = "backgroundDiseaseIds"

    # If diseaseId in schema, we need to drop it:
    drop_columns = [
        column
        for column in self.df.columns
        if column in [foreground_disease_column, background_disease_column]
    ]

    if len(drop_columns) > 0:
        self.df = self.df.drop(*drop_columns)

    # Normalise disease:
    normalised_disease = self._normalise_disease(
        "traitFromSourceMappedIds", foreground_disease_column, disease_map
    )
    normalised_background_disease = self._normalise_disease(
        "backgroundTraitFromSourceMappedIds", background_disease_column, disease_map
    )

    return StudyIndex(
        _df=(
            self.df.join(normalised_disease, on="studyId", how="left")
            .join(normalised_background_disease, on="studyId", how="left")
            # Updating disease columns:
            .withColumn(
                foreground_disease_column,
                f.when(
                    f.col(foreground_disease_column).isNull(), f.array()
                ).otherwise(f.col(foreground_disease_column)),
            )
            .withColumn(
                background_disease_column,
                f.when(
                    f.col(background_disease_column).isNull(), f.array()
                ).otherwise(f.col(background_disease_column)),
            )
            # Flagging gwas studies where no valid disease is avilable:
            .withColumn(
                "qualityControls",
                StudyIndex.update_quality_flag(
                    f.col("qualityControls"),
                    # Flagging all gwas studies with no normalised disease:
                    (f.size(f.col(foreground_disease_column)) == 0)
                    & (f.col("studyType") == "gwas"),
                    StudyQualityCheck.UNRESOLVED_DISEASE,
                ),
            )
        ),
        _schema=StudyIndex.get_schema(),
    )

validate_study_type() -> StudyIndex

Validating study type and flag unsupported types.

Returns:

Name Type Description
StudyIndex StudyIndex

with flagged studies with unsupported type.

Source code in src/gentropy/dataset/study_index.py
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
def validate_study_type(self: StudyIndex) -> StudyIndex:
    """Validating study type and flag unsupported types.

    Returns:
        StudyIndex: with flagged studies with unsupported type.
    """
    validated_df = (
        self.df
        # Flagging unsupported study types:
        .withColumn(
            "qualityControls",
            StudyIndex.update_quality_flag(
                f.col("qualityControls"),
                f.when(
                    (f.col("studyType") == "gwas")
                    | f.col("studyType").endswith("qtl"),
                    False,
                ).otherwise(True),
                StudyQualityCheck.UNKNOWN_STUDY_TYPE,
            ),
        )
    )
    return StudyIndex(_df=validated_df, _schema=StudyIndex.get_schema())

validate_target(target_index: GeneIndex) -> StudyIndex

Validating gene identifiers in the study index against the provided target index.

Parameters:

Name Type Description Default
target_index GeneIndex

gene index containing the reference gene identifiers (Ensembl gene identifiers).

required

Returns:

Name Type Description
StudyIndex StudyIndex

with flagged studies if geneId could not be validated.

Source code in src/gentropy/dataset/study_index.py
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
def validate_target(self: StudyIndex, target_index: GeneIndex) -> StudyIndex:
    """Validating gene identifiers in the study index against the provided target index.

    Args:
        target_index (GeneIndex): gene index containing the reference gene identifiers (Ensembl gene identifiers).

    Returns:
        StudyIndex: with flagged studies if geneId could not be validated.
    """
    gene_set = target_index.df.select("geneId", f.lit(True).alias("isIdFound"))

    # As the geneId is not a mandatory field of study index, we return if the column is not there:
    if "geneId" not in self.df.columns:
        return self

    validated_df = (
        self.df.join(gene_set, on="geneId", how="left")
        .withColumn(
            "isIdFound",
            f.when(
                (f.col("studyType") != "gwas") & f.col("isIdFound").isNull(),
                f.lit(False),
            ).otherwise(f.lit(True)),
        )
        .withColumn(
            "qualityControls",
            StudyIndex.update_quality_flag(
                f.col("qualityControls"),
                ~f.col("isIdFound"),
                StudyQualityCheck.UNRESOLVED_TARGET,
            ),
        )
        .drop("isIdFound")
    )

    return StudyIndex(_df=validated_df, _schema=StudyIndex.get_schema())

validate_unique_study_id() -> StudyIndex

Validating the uniqueness of study identifiers and flagging duplicated studies.

Returns:

Name Type Description
StudyIndex StudyIndex

with flagged duplicated studies.

Source code in src/gentropy/dataset/study_index.py
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
def validate_unique_study_id(self: StudyIndex) -> StudyIndex:
    """Validating the uniqueness of study identifiers and flagging duplicated studies.

    Returns:
        StudyIndex: with flagged duplicated studies.
    """
    validated_df = (
        self.df.withColumn(
            "isDuplicated",
            f.when(
                f.count("studyType").over(
                    Window.partitionBy("studyId").rowsBetween(
                        Window.unboundedPreceding, Window.unboundedFollowing
                    )
                )
                > 1,
                True,
            ).otherwise(False),
        )
        .withColumn(
            "qualityControls",
            StudyIndex.update_quality_flag(
                f.col("qualityControls"),
                f.col("isDuplicated"),
                StudyQualityCheck.DUPLICATED_STUDY,
            ),
        )
        .drop("isDuplicated")
    )
    return StudyIndex(_df=validated_df, _schema=StudyIndex.get_schema())

Schema

root
 |-- studyId: string (nullable = false)
 |-- projectId: string (nullable = false)
 |-- studyType: string (nullable = false)
 |-- traitFromSource: string (nullable = true)
 |-- traitFromSourceMappedIds: array (nullable = true)
 |    |-- element: string (containsNull = true)
 |-- diseaseIds: array (nullable = true)
 |    |-- element: string (containsNull = true)
 |-- geneId: string (nullable = true)
 |-- biosampleFromSourceId: string (nullable = true)
 |-- pubmedId: string (nullable = true)
 |-- publicationTitle: string (nullable = true)
 |-- publicationFirstAuthor: string (nullable = true)
 |-- publicationDate: string (nullable = true)
 |-- publicationJournal: string (nullable = true)
 |-- backgroundTraitFromSourceMappedIds: array (nullable = true)
 |    |-- element: string (containsNull = true)
 |-- backgroundDiseaseIds: array (nullable = true)
 |    |-- element: string (containsNull = true)
 |-- initialSampleSize: string (nullable = true)
 |-- nCases: integer (nullable = true)
 |-- nControls: integer (nullable = true)
 |-- nSamples: integer (nullable = true)
 |-- cohorts: array (nullable = true)
 |    |-- element: string (containsNull = true)
 |-- ldPopulationStructure: array (nullable = true)
 |    |-- element: struct (containsNull = false)
 |    |    |-- ldPopulation: string (nullable = true)
 |    |    |-- relativeSampleSize: double (nullable = true)
 |-- discoverySamples: array (nullable = true)
 |    |-- element: struct (containsNull = false)
 |    |    |-- sampleSize: integer (nullable = true)
 |    |    |-- ancestry: string (nullable = true)
 |-- replicationSamples: array (nullable = true)
 |    |-- element: struct (containsNull = false)
 |    |    |-- sampleSize: integer (nullable = true)
 |    |    |-- ancestry: string (nullable = true)
 |-- qualityControls: array (nullable = true)
 |    |-- element: string (containsNull = true)
 |-- analysisFlags: array (nullable = true)
 |    |-- element: string (containsNull = true)
 |-- summarystatsLocation: string (nullable = true)
 |-- hasSumstats: boolean (nullable = true)