Commit fc6577a2 authored by Daniel Falster's avatar Daniel Falster
Browse files

Reorgansie files

No related merge requests found
Showing with 0 additions and 206 deletions
+0 -206
File moved
File moved
source("email/email.R")
library("rjson")
sendEmail <- function(name, address, site, files = NA){
email.text = paste0(paste0("Dear ", name, ","),"
We are looking forward to the working group next week on competitive interactions and traits organised by Georges Kunstler at Macquarie University. You have provided one of the datasets we will be analysing. I am helping Georges prepare the data for analysis. To this end, we are preparing a file with metadata for each of the datasets, gathering together the various bits of information we have in a consistent format. Attached you find details about the ", site , " dataset, for which you are responsible.
We would appreciate if you could review the attached meta data information for the ", site , " dataset and get back to us, ideally within the next few days. Where possible, we are running analyses prior to the working group, so that we can begin to discuss results already on day one. For some datasets however, we do not yet have enough information to make this possible.
Several files are attached for you to review:
- The file 'site_description.pdf' combines all the meta data for your site and is built from the information contained in the other files. Thus, where data is absent, we ask you to edit the other data files to provide relevant data.
- The file '_site_details.JSON' contains information about your site, including contact details, data availability, site information, vegetation type, etc. This file can be opened in any text editor and information added where appropriate. In addtion to information described above, please also give a brief desciption about the 'contents' of each of the files listed. e.g. 'contains demographic data from field survey'.
- A separate csv file for each of the 'files' listed in JSON file, giving variable headings, units and variable descriptions for the data in each file. These csv files are used to create the tables in the pdf report. These csv files can edited either in a text editor or in Microsoft excel.
To give you a better idea of what a completed description would look like, I have also included a copy of the BCI site description, which is close to complete.
Please return to us updated versions of the the JSON and csv files as soon as possible.
With best regards,
Daniel Falster and Georges Kunstler
P: ARC Postdoctoral Research Fellow
E: daniel.falster@mq.edu.au
A: Biological Sciences E8C 160, Macquarie University NSW 2109, Australia
T: +61-2-9850-9258
W: http://www.falsters.net/daniel/\n\n\n")
email(content = email.text,
subject = paste("Preparing data for next week's working group at Macquarie University"),
to = address,
from="daniel.falster@mq.edu.au",
cc= "georges.kunstler@gmail.com",
files = files,
send=FALSE
)
}
# Generate tables for each file
for(DIR in dir("metadata", full.names = TRUE)){
cat(paste("\n", DIR))
json_file <- file.path(DIR,"_site_details.JSON")
details <- fromJSON(paste(readLines(json_file), collapse=""))
files <- list.files(DIR, recursive = TRUE, full.names = TRUE)
sendEmail(name =details$contact$name, address=details$contact$email, site = details$dataset$name, files = c(files, "site_description_BCI.pdf"))
}
# Removes bad files endings on csv data from saving with Microsoft Excel in mac
find . -name '*.csv' | xargs perl -pi -e 's/\r/\n/g'
# Scripts used to create folders and files lists for metadata
# caution - if rerun, will overwrite existing files
# Make directory for all input directories
for(d in gsub("Data", "", dir("data/raw")))
dir.create(file.path("ms/metadata", d))
# List all files in raw data directories
for(d in dir("data/raw", full.names = TRUE)){
pathout <- file.path("ms/metadata", gsub("Data", "", basename(d)))
files <- gsub(paste0(d,"/"), "", dir(d, recursive=TRUE, full.names = TRUE))
files <- files[ tools::file_ext(files) %in% c("csv", "txt") ]
sep= rep("t", length(files))
sep[tools::file_ext(files) =="csv"] ="c"
write.csv(data.frame(filename= files, sep = sep, skip=0, contents=" "), file=file.path(pathout, "_file_index.csv"), row.names=FALSE, quote=FALSE)
}
make_meta_data_tables <- function(d){
files <- read.csv(file.path(d,"_file_index.csv"), stringsAsFactors=FALSE)
for(i in 1:length(files[,1])){
cat(paste(files[i,"sep"], " "))
sep = "\t"
if(files[i,"sep"] == "c") sep= ","
if(files[i,"sep"] == ";") sep= ";"
if(files[i,"sep"] == " ") sep= " "
infile <- file.path(gsub("ms/metadata/", "data/raw/Data", d), files[i,"filename"])
data <- read.table(infile, sep=sep, stringsAsFactors=FALSE, fill=TRUE, header=TRUE)
write.csv(data.frame(use=0, var=names(data), units="", description=""), quote=FALSE, row.names=FALSE,
file = file.path(d, "files",
paste0(tools::file_path_sans_ext(gsub("/", "_", files[i,"filename"])), ".csv")))
}
}
# Generate tables for each file
for(d in dir("ms/metadata", full.names = TRUE)[3]){
cat(paste("\n", d))
make_meta_data_tables(d)
}
l <- list()
for(i in 1:length(details[,1]) )
l[[details[i,1]]]<- details[i,2]
# Generate tables for each file
f <- "new.JSON"
cat("", file = f)
for(d in dir("metadata", full.names = TRUE) ){
cat(paste(d, "\n\n\tfiles: "), file = f, append = TRUE)
x<-read.csv(file.path(d,"_file_index.csv"), stringsAsFactors=FALSE)
l <- list()
for(i in 1:length(x[,1]) )
l[[x[i,1]]]<- as.list(x[1,2:4])
cat(toJSON(l), file = f, append = TRUE)
}
#################################################### SPECIES LIST FOR WORKSHOP
################################# READ SPECIES LIST I HAVE NOW FOR EACH SITES NON TROPICAL (ASSUMING THAT
################################# TROPICAL SITES COMES WITH TRAITS DATA
############################# 3 US based on FIA full species list
Data.Species.FIA <- read.csv("./data/species.list/REF_SPECIES.CSV", stringsAsFactors = FALSE,
na.strings = "")
## head(Data.Species.FIA) merge genus & species
Data.Species.FIA$sp <- gsub("spp.", "sp", paste(Data.Species.FIA$GENUS, Data.Species.FIA$SPECIES))
Species.FIA.short <- Data.Species.FIA$sp[Data.Species.FIA$SPCD < 6000]
Species.FIA.long <- Data.Species.FIA$sp
#################################### US SPECIES Based on Burns and Honkala
BurnsAndHonkalaSpeciesSummary <- read.csv("./data/species.list/BurnsAndHonkalaSpeciesSummary.csv",
stringsAsFactors = FALSE, na.strings = "", sep = ";")
## head(BurnsAndHonkalaSpeciesSummary) clean species name
fun.get.sp <- function(x) gsub(paste(" ", gsub("^([a-zA-Z]* [a-zA-Z]* )", "", x),
sep = ""), "", x, fixed = TRUE)
BurnsAndHonkalaSpeciesSummary$sp <- sapply(BurnsAndHonkalaSpeciesSummary[, "Scientific_name"],
fun.get.sp)
Species.Burns <- BurnsAndHonkalaSpeciesSummary$sp
######################################### canada check http://en.wikipedia.org/wiki/List_of_trees_of_Canada
Data.canadian.species <- read.csv("./data/species.list/canadian.species.csv", stringsAsFactors = FALSE)
Species.Canada <- Data.canadian.species$species
######################### SPAIN BASED ON TABLE PROVIDED BY MIGUEL
Data.Species.Spain <- read.csv("./data/species.list/List_120_SPP.csv", stringsAsFactors = FALSE)
## head(Data.Species.Spain)
Species.Spain <- Data.Species.Spain$Scoentific.Name
######################################### SPECIE Sin Swizerland not in FRANCE
Species.Swiss <- c("Pinus montana", "Ulmus scabra")
#################################### 3 SWEDEN
Data.Sweden <- read.csv("./data/species.list/Swedish_NFI_tree_species.csv", sep = ";",
stringsAsFactors = FALSE)
Species.Sweden <- Data.Sweden$Species
########################### NEW ZEALAND
load("./data/species.list/vec.code.nvs.Rdata") ### load list of species code from the NVS data I used to calibrate SORTIE
Data.NVS.species <- read.csv("./data/species.list/CurrentNVSNames.csv", sep = ";",
stringsAsFactors = FALSE)
table.NVS.tree <- Data.NVS.species[Data.NVS.species$NVS.Code %in% vec.code.nvs, ]
write.csv(table.NVS.tree, file = "./data/species.list/table.NVS.tree.csv")
Species.NZ <- table.NVS.tree$Species.Name
########################### NSW
Data.NSW <- read.csv("./data/species.list/Sub-trop_RF_trees.csv", sep = ";", stringsAsFactors = FALSE)
Species.NSW <- Data.NSW$Species..trees...10cm.dbhob.
########################################### LOAD TRY SPECIES
try.species <- read.csv("./data/species.list/SPECIES.TRY.csv", stringsAsFactors = FALSE,
header = TRUE)
## head(try.species,20) dim(try.species)
## genus already asked to try
genus.asked.try <- read.csv("./data/species.list/genus.asked.try.csv")$genus.asked.try
#### LIST OF ALL SPECIES with short FIA
species.vector <- unique(c(Species.Spain, Species.Canada, Species.Burns, Species.Swiss,
Species.Sweden, Species.NZ, Species.NSW, Species.FIA.short))
#
### get genus
fun.get.genus <- function(x) gsub(paste(" ", gsub("^([a-zA-Z]* )", "", x), sep = ""),
"", x, fixed = TRUE)
Genus.tot <- unique(sapply(species.vector, fun.get.genus))
## all FIA data species.vector2 <-
## unique(c(Species.Spain,Species.Canada,Species.Burns
## ,Species.Swiss,Species.Sweden,as.character(Species.NZ),Species.NSW,Species.FIA.long))
## Genus.tot2 <- unique(sapply(species.vector2,fun.get.genus))
#### GENUS TO ASK
GENUS.to.ASK <- Genus.tot[!(Genus.tot %in% genus.asked.try)]
## GENUS.to.ASK2 <- Genus.tot2[!(Genus.tot2 %in% genus.asked.try)]
## CHECK which genus are in TRY
GENUS.TRY <- unique(sapply(try.species$AccSpeciesName, fun.get.genus))
############################# 3
genus.list.short <- sort(GENUS.to.ASK[(GENUS.to.ASK %in% GENUS.TRY)])
write.csv(as.data.frame(genus.list.short), file = "./data/process/genus.list.try.csv")
sort(GENUS.to.ASK[!(GENUS.to.ASK %in% GENUS.TRY)])
## genus.lis.long <- sort(GENUS.to.ASK2[(GENUS.to.ASK2 %in% GENUS.TRY)])
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment