---
title: "Supporting data for the TCGAbiolinksGUI package"
author: "Tiago Chedraoui Silva, Tathiane Malta, Houtan Noushmehr"
date: "`r Sys.Date()`"
output: 
    BiocStyle::html_document:
      highlight: tango
      toc: yes
      fig_caption: yes     
      toc_depth: 3
      toc_float:
        collapsed: yes
      number_sections: true
    editor_options: 
      chunk_output_type: inline
references:
vignette: >
  \usepackage[utf8]{inputenc}
  %\VignetteIndexEntry{Supporting data for the TCGAbiolinksGUI package}
  %\VignetteEngine{knitr::rmarkdown}
editor_options: 
  chunk_output_type: inline
bibliography: bibliography.bib    
---   

# Introduction

This document provides an introduction of the `r BiocStyle::Biocpkg("TCGAbiolinksGUI.data")`, which contains 
supporting data for `r BiocStyle::Biocpkg("TCGAbiolinksGUI")` [@silva2017tcgabiolinksgui]. 

This package contains the following objects:

- For Glioma Classifier function
  - glioma.gcimp.model  
  - glioma.idh.model
  - glioma.idhmut.model
  - glioma.idhwt.model 
- For linkedOmics database:
  - linkedOmicsData

## Installing TCGAbiolinksGUI.data

You can install the package from Bioconductor:
```{r, eval = FALSE}
if (!requireNamespace("BiocManager", quietly=TRUE))
    install.packages("BiocManager")
BiocManager::install(("TCGAbiolinksGUI.data")
```

Or from GitHub:
```{r, eval = FALSE}
BiocManager::install("BioinformaticsFMRP/TCGAbiolinksGUI.data")
```

Load the package:

```{r, eval = TRUE}
library(TCGAbiolinksGUI.data)
```


# Contents

## Creating list of GDC projects

The code below access the NCI's Genomic Data Commons (GDC) and get the list of available datasets for The Cancer Genome Atlas (TCGA) and Therapeutically Applicable Research to Generate Effective Treatments (TARGET) projects.

```{r, eval=FALSE, include=TRUE}
# Defining parameters
getGDCdisease <- function(){
  projects <- TCGAbiolinks:::getGDCprojects()
  projects <- projects[projects$id != "FM-AD",]
  disease <-  projects$project_id
  idx <- grep("disease_type",colnames(projects))
  names(disease) <-  paste0(projects[[idx]], " (",disease,")")
  disease <- disease[sort(names(disease))]
  return(disease)
}
```

This data is in saved in the GDCdisease object.
```{r}
data(GDCdisease)
DT::datatable(as.data.frame(GDCdisease))
```

## List of MAF files

The code below downloads a manifest of open TCGA MAF files available in the NCI's Genomic Data Commons (GDC).
```{r, eval=FALSE, include=TRUE}
getMafTumors <- function(){
  root <- "https://gdc-api.nci.nih.gov/data/"
  maf <- fread("https://gdc-docs.nci.nih.gov/Data/Release_Notes/Manifests/GDC_open_MAFs_manifest.txt",
               data.table = FALSE, verbose = FALSE, showProgress = FALSE)
  tumor <- unlist(lapply(maf$filename, function(x){unlist(str_split(x,"\\."))[2]}))
  proj <- TCGAbiolinks:::getGDCprojects()
  
  disease <-  gsub("TCGA-","",proj$project_id)
  idx <- grep("disease_type",colnames(proj))
  names(disease) <-  paste0(proj[[idx]], " (",proj$project_id,")")
  disease <- sort(disease)
  ret <- disease[disease %in% tumor]
  return(ret)
}
```

This data is in saved in the maf.tumor object.
```{r}
data(maf.tumor)
DT::datatable(as.data.frame(maf.tumor))
```


## Creating Training models

Based on the article data from the article "Molecular Profiling Reveals Biologically Discrete Subsets and Pathways of Progression in Diffuse Glioma" (www.cell.com/cell/abstract/S0092-8674(15)01692-X) [@Cell] we created a training model to predict Glioma classes based on DNA methylation signatures.

First, we  will load the required libraries, control random number generation by specifying a seed and register the number of cores for parallel evaluation.
```{r, eval=FALSE, include=TRUE}
library(readr)
library(readxl)
library(dplyr)
library(caret)
library(randomForest)
library(doMC)
library(e1071)

# Control random number generation
set.seed(210) # set a seed to RNG

# register number of cores to be used for parallel evaluation
registerDoMC(cores = parallel::detectCores())
```

The next steps will download the DNA methylation matrix from the article: the DNA methylation data for glioma samples, samples metadata, and 
DNA methylation signatures. 

```{r, eval=FALSE, include=TRUE}
file <- "https://tcga-data.nci.nih.gov/docs/publications/lgggbm_2016/LGG.GBM.meth.txt"
if(!file.exists(basename(file))) downloader::download(file,basename(file))
LGG.GBM <- as.data.frame(readr::read_tsv(basename(file)))
rownames(LGG.GBM) <- LGG.GBM$Composite.Element.REF
idx <- grep("TCGA",colnames(LGG.GBM))
colnames(LGG.GBM)[idx] <- substr(colnames(LGG.GBM)[idx], 1, 12) # reduce complete barcode to sample identifier (first 12 characters)  
```

We will get metadata with samples molecular subtypes from the paper:  (www.cell.com/cell/abstract/S0092-8674(15)01692-X) [@Cell]

```{r, eval=FALSE, include=TRUE}
file <- "http://www.cell.com/cms/attachment/2045372863/2056783242/mmc2.xlsx"
if(!file.exists(basename(file))) downloader::download(file,basename(file))
metadata <-  readxl::read_excel(basename(file), sheet = "S1A. TCGA discovery dataset", skip = 1)
DT::datatable(metadata[,c("Case",
                          "Pan-Glioma DNA Methylation Cluster",
                          "Supervised DNA Methylation Cluster",
                          "IDH-specific DNA Methylation Cluster")])
```

Probes metadata information are downloaded from http://zwdzwd.io/InfiniumAnnotation
This will be used  to remove probes that should be masked from the training.
```{r, eval=FALSE, include=TRUE}
file <- "http://zwdzwd.io/InfiniumAnnotation/current/EPIC/EPIC.manifest.hg38.rda"
if(!file.exists(basename(file))) downloader::download(file,basename(file))
load(basename(file))
```

Retrieve probe signatures from the paper.
```{r, eval=FALSE, include=TRUE}
file <- "https://tcga-data.nci.nih.gov/docs/publications/lgggbm_2015/PanGlioma_MethylationSignatures.xlsx"
if(!file.exists(basename(file))) downloader::download(file,basename(file))
```


With the data and metadata available we will create one model for each signature. 
The code below selects the DNA methylation values for a given set of signatures (probes)
and uses the classification of each sample to create a Random forest model.
Each model is described in the next subsections.

### RF to classify between IDHmut and IDHwt

| Parameters  | Values   |
|---|---|
| trainingset  | whole TCGA panglioma cohort  |
| probes signature  | 1,300 pan-glioma tumor specific  |
| groups to be classified | LGm1, LGm2, LGm3, LGm4, LGm5, LGm6  |
| metadata column | Pan-Glioma DNA Methylation Cluster |

We will start by preparing the training data. We will select the probes signatures for the group classification from the excel file
(for this case the 1,300 probes) and the samples that belong to the groups we want to create our model (in this case "LGm2" "LGm5" "LGm3" "LGm4" "LGm1" "LGm6").

```{r, eval=FALSE, include=TRUE}
sheet <- "1,300 pan-glioma tumor specific"
trainingset <- grep("mut|wt",unique(metadata$`Pan-Glioma DNA Methylation Cluster`),value = T)
trainingcol <- c("Pan-Glioma DNA Methylation Cluster")
```

The DNA methylation matrix will be subset to the DNA methylation
signatures and samples with classification.
```{r, eval=FALSE, include=TRUE}
plat <- "EPIC"
signature.probes <-  read_excel("PanGlioma_MethylationSignatures.xlsx",  sheet = sheet)  %>% pull(1) 
samples <- dplyr::filter(metadata, `IDH-specific DNA Methylation Cluster` %in% trainingset)
RFtrain <- LGG.GBM[signature.probes, colnames(LGG.GBM) %in% as.character(samples$Case)] %>% na.omit 
```

Probes that should be masked, will be removed.
```{r, eval=FALSE, include=TRUE}
RFtrain <- RFtrain[!EPIC.manifest.hg38[rownames(RFtrain)]$MASK.general,]
```

We will merge the samples with their classification. In the end, we will have samples in the row, and probes and classification as columns.

```{r, eval=FALSE, include=TRUE}
trainingdata <- t(RFtrain)
trainingdata <- merge(trainingdata, metadata[,c("Case", trainingcol[model])], by.x=0,by.y="Case", all.x=T)
rownames(trainingdata) <- as.character(trainingdata$Row.names)
trainingdata$Row.names <- NULL
```

After the data prepared we will start the RF training by selecting tuning the mtry argument, which defines 
the number of variables randomly sampled as candidates at each split. We will use the function tuneRF to  optimal mtry 
from values going from sqrt(p) where p is number of probes in the data up to 2 * sqrt(p)
(see tuneRF function for more information). RF will use all mtry values and use the one that produced the best model.

```{r, eval=FALSE, include=TRUE}
nfeat <- ncol(trainingdata)
trainingdata[,trainingcol] <-  factor(trainingdata[,trainingcol])
mtryVals <- floor(sqrt(nfeat))
for(i in floor(seq(sqrt(nfeat), nfeat/2, by = 2 * sqrt(nfeat)))) {
  print(i)
  x <- as.data.frame(tuneRF(trainingdata[,-grep(trainingcol[model],colnames(trainingdata))], 
                            trainingdata[,trainingcol[model]], 
                            stepFactor=2,
                            plot= FALSE,
                            mtryStart = i))
  mtryVals <- unique(c(mtryVals, x$mtry[which (x$OOBError  == min(x$OOBError))]))
}
mtryGrid <- data.frame(.mtry=mtryVals)
```

We will use the training data to create our Random forest model. First, we will set up a repeated k-fold cross-validation.
We set k=10, which will split the data into 10 equal size sets. Of the 10 set, one is retained as the validation data for testing the model, and the remaining 9 sets are used as training data. The cross-validation process is then repeated 10 times (the folds), each time using a different set as the validation data. The 10 results from the folds can then be averaged (or otherwise combined) to produce a single estimation.
This procedure will be repeated 10 times, with different 10 equal size sets. The final model accuracy is taken as the mean from the number of repeats.
```{r, eval=FALSE, include=TRUE}
fitControl <- trainControl(method = "repeatedcv",
                           number = 10,
                           verboseIter = TRUE,
                           repeats = 10)
```

Create our Random forest model using the train frunction from the caret package.
```{r, eval=FALSE, include=TRUE}
glioma.idh.model <- train(y = trainingdata[,trainingcol], # variable to be trained on
                          x = trainingdata[,-grep(trainingcol,colnames(trainingdata))], # Daat labels
                          data = trainingdata, # Data we are using
                          method = "rf", # Method we are using
                          trControl = fitControl, # How we validate
                          ntree = 5000, # number of trees
                          importance = TRUE, 
                          tuneGrid = mtryGrid, # set mtrys, the value that procuded a better model is used
)
```

The IDH model are saved in *glioma.idh.model* object.
```{r}
data(glioma.idh.model)
glioma.idh.model
```


### RF to classify IDHmut specific clusters
| Parameters  | Values   |
|---|---|
| trainingset  |  TCGA IDHmut only  |
| probes signature  | 1,308 IDHmutant tumor specific  |
| groups to be classified | IDHmut-K1, IDHmut-K2, IDHmut-K3  |
| metadata column | IDH-specific DNA Methylation Cluster |

To produce the model we will use the same code above but we will change the training data (probes and labels are different)
```{r, eval=FALSE, include=TRUE}
sheet <- "1,308 IDHmutant tumor specific "
trainingset <- grep("mut",unique(metadata$`IDH-specific DNA Methylation Cluster`),value = T)
trainingcol <- "IDH-specific DNA Methylation Cluster"
```

The IDHmut model  are saved in  glioma.idhmut.model object.
```{r}
data(glioma.idhmut.model)
glioma.idhmut.model
```


### RF to classify between G-CIMP-low and G-CIMP-high 
| Parameters  | Values   |
|---|---|
| trainingset  |  TCGA IDHmut-K1 and IDHmut-K2 only  |
| probes signature  | 163  probes that define each TC  |
| groups to be classified | G-CIMP-low, G-CIMP-high  |
| metadata column | Supervised DNA Methylation Cluster |

To produce the model we will use the same code above but we will change the training data (probes and labels are different).
```{r, eval=FALSE, include=TRUE}
sheet <- "163  probes that define each TC"
trainingset <- c("IDHmut-K1","IDHmut-K2")
trainingcol <- "Supervised DNA Methylation Cluster"
```

The G-CIMP model was saved in  glioma.gcimp.model object.
```{r}
data("glioma.gcimp.model")
glioma.gcimp.model
```

### RF to classify IDHwt specific clusters 

| Parameters  | Values   |
|---|---|
| trainingset  | TCGA IDHwt only  |
| probes signature  | 914 IDHwildtype tumor specific  |
| groups to be classified | IDHwt-K1, IDHwt-K2, IDHwt-K3  |
| metadata column | IDH-specific DNA Methylation Cluster |

Note: In this case, samples classified into IDHwt-K3 
should be further subdivided by grade.

To produce the model we will use the same code above but we will change the training data (probes and labels are different)
```{r, eval=FALSE, include=TRUE}
sheet <- "914 IDHwildtype tumor specific "
trainingset <- grep("wt",unique(metadata$`IDH-specific DNA Methylation Cluster`),value = T))
trainingcol <- "IDH-specific DNA Methylation Cluster"
```

The IDHwt specific model which classifies are saved in  glioma.idh.model object.
```{r}
data("glioma.idhwt.model")
glioma.idhwt.model
```

## EPIC probes to remove

The list of probes to be removed from EPIC array due to differences in library versions were taken from
https://support.illumina.com/downloads/infinium-methylationepic-v1-0-product-files.html (Infinium MethylationEPIC v1.0 Missing Legacy CpG (B3 vs. B2) Annotation File)

```{r}
data("probes2rm")
head(probes2rm)
```

## Parsing linked omics database

The code below will parse all links from omicsLinks website.

```{R, eval = FALSE}
scraplinks <- function(url){
    # Create an html document from the url
    webpage <- xml2::read_html(url)
    # Extract the URLs
    url_ <- webpage %>%
        rvest::html_nodes("a") %>%
        rvest::html_attr("href")
    # Extract the link text
    link_ <- webpage %>%
        rvest::html_nodes("a") %>%
        rvest::html_text()
    tb <- tibble::tibble(link = link_, url = url_)
    tb <- tb %>% dplyr::filter(tb$link == "Download")
    return(tb)
}

library(htmltab)
library(dplyr)
library(tidyr)
root <- "http://linkedomics.org"
root.download <- file.path(root,"data_download")
linkedOmics <- htmltab(paste0(root,"/login.php#dataSource"))
linkedOmics <- linkedOmics %>% unite(col = "download_page","Cohort Source","Cancer ID", remove = FALSE,sep = "-")
linkedOmics.data <- plyr::adply(linkedOmics$download_page,1,function(project){
    url <- file.path(root.download,project)
    tryCatch({
        tb <- cbind(tibble::tibble(project),htmltab(url),scraplinks(url))
        tb$Link <- tb$link <- NULL
        tb$dataset <- gsub(" \\(.*","",tb$`OMICS Dataset`)
        tb
    }, error = function(e) {
        message(e)
        return(NULL)
    }
    )
},.progress = "time",.id = NULL)
usethis::use_data(linkedOmics.data,internal = FALSE,compress = "xz")
```

# Session Information

******
```{r sessionInfo}
sessionInfo()
```

# References