Reorgansie files

ms/email/send-email.R

-
-source("email/email.R")
-library("rjson")
-
-sendEmail <- function(name, address, site, files = NA){
-  
-  email.text  =  paste0(paste0("Dear ", name, ","),"
-  	
-We are looking forward to the working group next week on competitive interactions and traits organised by Georges Kunstler at Macquarie University. You have provided one of the datasets we will be analysing. I am helping Georges prepare the data for analysis. To this end, we are preparing a file with metadata for each of the datasets, gathering together the various bits of information we have in a consistent format. Attached you find details about the ", site , " dataset, for which you are responsible.
-
-We would appreciate if you could review the attached meta data information for the ", site , " dataset and get back to us, ideally within the next few days. Where possible, we are running analyses prior to the working group, so that we can begin to discuss results already on day one. For some datasets however, we do not yet have enough information to make this possible.
-
-Several files are attached for you to review:
-- The file 'site_description.pdf' combines all the meta data for your site and is built from the information contained in the other files. Thus, where data is absent, we ask you to edit the other data files to provide relevant data.
-- The file '_site_details.JSON' contains information about your site, including contact details, data availability, site information, vegetation type, etc. This file can be opened in any text editor and information added where appropriate. In addtion to information described above, please also give a brief desciption about the 'contents' of each of the files listed. e.g. 'contains demographic data from field survey'.
-- A separate csv file for each of the 'files' listed in JSON file, giving variable headings, units and variable descriptions for the data in each file. These csv files are used to create the tables in the pdf report. These csv files can edited either in a text editor or in Microsoft excel.
-
-To give you a better idea of what a completed description would look like, I have also included a copy of the BCI site description, which is close to complete.
-
-Please return to us updated versions of the the JSON and csv files as soon as possible.
-
-With best regards,
-Daniel Falster and Georges Kunstler
-
-P: ARC Postdoctoral Research Fellow
-E: daniel.falster@mq.edu.au
-A: Biological Sciences E8C 160, Macquarie University NSW 2109, Australia
-T: +61-2-9850-9258
-W: http://www.falsters.net/daniel/\n\n\n")
-  
-email(content =   email.text,
-        subject = paste("Preparing data for next week's working group at Macquarie University"),
-        to =  address, 
-        from="daniel.falster@mq.edu.au",  
-        cc= "georges.kunstler@gmail.com",
-        files = files, 
-        send=FALSE
-  )
-}  
-
-# Generate tables for each file
-for(DIR in dir("metadata", full.names = TRUE)){
-	cat(paste("\n", DIR))	
-	
-	json_file <- file.path(DIR,"_site_details.JSON")
-	details <- fromJSON(paste(readLines(json_file), collapse=""))
-	files <- list.files(DIR, recursive = TRUE, full.names = TRUE)
-	sendEmail(name =details$contact$name, address=details$contact$email, site = details$dataset$name, files = c(files, "site_description_BCI.pdf"))
-}
-

ms/tmp/cleanCSV.sh

-# Removes bad files endings on csv data from saving with Microsoft Excel in mac
-find . -name '*.csv' | xargs perl -pi -e 's/\r/\n/g'

ms/tmp/create_folders.R

-
-# Scripts used to create folders and files lists for metadata 
-# caution - if rerun, will overwrite existing files
-
-# Make directory for all input directories
-for(d in gsub("Data", "", dir("data/raw")))
-	dir.create(file.path("ms/metadata", d))
-
-# List all files in raw data directories
-
-for(d in dir("data/raw", full.names = TRUE)){
-	pathout <- file.path("ms/metadata", gsub("Data", "", basename(d)))
-	files <-  gsub(paste0(d,"/"), "", dir(d, recursive=TRUE, full.names = TRUE))
-	files <- files[ tools::file_ext(files) %in% c("csv", "txt") ]
-	sep= rep("t", length(files))
-	sep[tools::file_ext(files) =="csv"] ="c"
-	write.csv(data.frame(filename= files, sep = sep, skip=0, contents=" "), file=file.path(pathout, "_file_index.csv"), row.names=FALSE, quote=FALSE)	
-}
-
-make_meta_data_tables <- function(d){	
-	files <- read.csv(file.path(d,"_file_index.csv"), stringsAsFactors=FALSE)
-	for(i in 1:length(files[,1])){
-		cat(paste(files[i,"sep"], " "))
-		sep = "\t"
-		if(files[i,"sep"] == "c") sep= ","
-		if(files[i,"sep"] == ";") sep= ";"
-		if(files[i,"sep"] == " ") sep= " "
-		infile <- file.path(gsub("ms/metadata/", "data/raw/Data", d), files[i,"filename"])
-		data <- read.table(infile, sep=sep, stringsAsFactors=FALSE, fill=TRUE, header=TRUE)
-		write.csv(data.frame(use=0, var=names(data), units="", description=""), quote=FALSE, row.names=FALSE,
-			file = file.path(d, "files",
-				paste0(tools::file_path_sans_ext(gsub("/", "_", files[i,"filename"])), ".csv")))				
-	}
-}
-
-
-# Generate tables for each file
-for(d in dir("ms/metadata", full.names = TRUE)[3]){
-	cat(paste("\n", d))
-	make_meta_data_tables(d)		
-}
-
-l <- list()
-for(i in 1:length(details[,1]) )
-	l[[details[i,1]]]<- details[i,2]
-	
-	
-# Generate tables for each file
-f <- "new.JSON"
-cat("", file = f)
-	
-for(d in dir("metadata", full.names = TRUE) ){
-	cat(paste(d, "\n\n\tfiles: "), file = f, append = TRUE)
-	x<-read.csv(file.path(d,"_file_index.csv"), stringsAsFactors=FALSE)
-	l <- list()
-	for(i in 1:length(x[,1]) )
-		l[[x[i,1]]]<- as.list(x[1,2:4])
-	cat(toJSON(l), file = f, append = TRUE)
-	}
-	
-	
-	
-

species.list.R

-#################################################### SPECIES LIST FOR WORKSHOP
-
-
-################################# READ SPECIES LIST I HAVE NOW FOR EACH SITES NON TROPICAL (ASSUMING THAT
-################################# TROPICAL SITES COMES WITH TRAITS DATA
-
-############################# 3 US based on FIA full species list
-Data.Species.FIA <- read.csv("./data/species.list/REF_SPECIES.CSV", stringsAsFactors = FALSE, 
-    na.strings = "")
-## head(Data.Species.FIA) merge genus & species
-Data.Species.FIA$sp <- gsub("spp.", "sp", paste(Data.Species.FIA$GENUS, Data.Species.FIA$SPECIES))
-Species.FIA.short <- Data.Species.FIA$sp[Data.Species.FIA$SPCD < 6000]
-Species.FIA.long <- Data.Species.FIA$sp
-
-#################################### US SPECIES Based on Burns and Honkala
-BurnsAndHonkalaSpeciesSummary <- read.csv("./data/species.list/BurnsAndHonkalaSpeciesSummary.csv", 
-    stringsAsFactors = FALSE, na.strings = "", sep = ";")
-## head(BurnsAndHonkalaSpeciesSummary) clean species name
-fun.get.sp <- function(x) gsub(paste(" ", gsub("^([a-zA-Z]* [a-zA-Z]* )", "", x), 
-    sep = ""), "", x, fixed = TRUE)
-BurnsAndHonkalaSpeciesSummary$sp <- sapply(BurnsAndHonkalaSpeciesSummary[, "Scientific_name"], 
-    fun.get.sp)
-Species.Burns <- BurnsAndHonkalaSpeciesSummary$sp
-
-
-######################################### canada check http://en.wikipedia.org/wiki/List_of_trees_of_Canada
-Data.canadian.species <- read.csv("./data/species.list/canadian.species.csv", stringsAsFactors = FALSE)
-Species.Canada <- Data.canadian.species$species
-
-######################### SPAIN BASED ON TABLE PROVIDED BY MIGUEL
-Data.Species.Spain <- read.csv("./data/species.list/List_120_SPP.csv", stringsAsFactors = FALSE)
-## head(Data.Species.Spain)
-Species.Spain <- Data.Species.Spain$Scoentific.Name
-######################################### SPECIE Sin Swizerland not in FRANCE
-Species.Swiss <- c("Pinus montana", "Ulmus scabra")
-
-#################################### 3 SWEDEN
-Data.Sweden <- read.csv("./data/species.list/Swedish_NFI_tree_species.csv", sep = ";", 
-    stringsAsFactors = FALSE)
-Species.Sweden <- Data.Sweden$Species
-
-########################### NEW ZEALAND
-load("./data/species.list/vec.code.nvs.Rdata")  ### load list of species code from the NVS data I used to calibrate SORTIE
-Data.NVS.species <- read.csv("./data/species.list/CurrentNVSNames.csv", sep = ";", 
-    stringsAsFactors = FALSE)
-
-table.NVS.tree <- Data.NVS.species[Data.NVS.species$NVS.Code %in% vec.code.nvs, ]
-write.csv(table.NVS.tree, file = "./data/species.list/table.NVS.tree.csv")
-Species.NZ <- table.NVS.tree$Species.Name
-
-########################### NSW
-Data.NSW <- read.csv("./data/species.list/Sub-trop_RF_trees.csv", sep = ";", stringsAsFactors = FALSE)
-Species.NSW <- Data.NSW$Species..trees...10cm.dbhob.
-
-
-########################################### LOAD TRY SPECIES
-try.species <- read.csv("./data/species.list/SPECIES.TRY.csv", stringsAsFactors = FALSE, 
-    header = TRUE)
-## head(try.species,20) dim(try.species)
-
-## genus already asked to try
-genus.asked.try <- read.csv("./data/species.list/genus.asked.try.csv")$genus.asked.try
-
-#### LIST OF ALL SPECIES with short FIA
-species.vector <- unique(c(Species.Spain, Species.Canada, Species.Burns, Species.Swiss, 
-    Species.Sweden, Species.NZ, Species.NSW, Species.FIA.short))
-# 
-
-### get genus
-fun.get.genus <- function(x) gsub(paste(" ", gsub("^([a-zA-Z]* )", "", x), sep = ""), 
-    "", x, fixed = TRUE)
-Genus.tot <- unique(sapply(species.vector, fun.get.genus))
-
-## all FIA data species.vector2 <-
-## unique(c(Species.Spain,Species.Canada,Species.Burns
-## ,Species.Swiss,Species.Sweden,as.character(Species.NZ),Species.NSW,Species.FIA.long))
-
-## Genus.tot2 <- unique(sapply(species.vector2,fun.get.genus))
-
-
-#### GENUS TO ASK
-GENUS.to.ASK <- Genus.tot[!(Genus.tot %in% genus.asked.try)]
-## GENUS.to.ASK2 <- Genus.tot2[!(Genus.tot2 %in% genus.asked.try)]
-
-## CHECK which genus are in TRY
-GENUS.TRY <- unique(sapply(try.species$AccSpeciesName, fun.get.genus))
-############################# 3
-genus.list.short <- sort(GENUS.to.ASK[(GENUS.to.ASK %in% GENUS.TRY)])
-write.csv(as.data.frame(genus.list.short), file = "./data/process/genus.list.try.csv")
-sort(GENUS.to.ASK[!(GENUS.to.ASK %in% GENUS.TRY)])
-## genus.lis.long <- sort(GENUS.to.ASK2[(GENUS.to.ASK2 %in% GENUS.TRY)])