creating metadata files for each dataset

.gitignore

@@ -4,7 +4,6 @@ data/*
 figs
 *.xls
 *.xlsx
-*.csv
 .Rhistory
 *.pdf
 *.doc

ms/create_folders.R

+
+# Scripts used to create folders and files lists for metadata 
+# caution - if rerun, will overwrite existing files
+
+# Make directory for all input directories
+for(d in gsub("Data", "", dir("data/raw")))
+	dir.create(file.path("ms/metadata", d))
+
+# List all files in raw data directories
+
+for(d in dir("data/raw", full.names = TRUE)){
+	pathout <- file.path("ms/metadata", gsub("Data", "", basename(d)))
+	files <-  gsub(paste0(d,"/"), "", dir(d, recursive=TRUE, full.names = TRUE))
+	files <- files[ tools::file_ext(files) %in% c("csv", "txt") ]
+	sep= rep("t", length(files))
+	sep[tools::file_ext(files) =="csv"] ="c"
+	write.csv(data.frame(filename= files, sep = sep, skip=0, contents=" "), file=file.path(pathout, "_file_index.csv"), row.names=FALSE, quote=FALSE)	
+}
+
+make_meta_data_tables <- function(d){	
+	files <- read.csv(file.path(d,"_file_index.csv"), stringsAsFactors=FALSE)
+	for(i in 1:length(files[,1])){
+		cat(paste(files[i,"sep"], " "))
+		sep = "\t"
+		if(files[i,"sep"] == "c") sep= ","
+		if(files[i,"sep"] == ";") sep= ";"
+		if(files[i,"sep"] == " ") sep= " "
+		infile <- file.path(gsub("ms/metadata/", "data/raw/Data", d), files[i,"filename"])
+		data <- read.table(infile, sep=sep, stringsAsFactors=FALSE, fill=TRUE, header=TRUE)
+		write.csv(data.frame(use=0, var=names(data), units="", description=""), quote=FALSE, row.names=FALSE,
+			file = file.path(d, "files",
+				paste0(tools::file_path_sans_ext(gsub("/", "_", files[i,"filename"])), ".csv")))				
+	}
+}
+
+
+# Generate tables for each file
+for(d in dir("ms/metadata", full.names = TRUE)[7:12]){
+	cat(paste("\n", d))
+	make_meta_data_tables(d)		
+}

ms/metadata/BCI/_file_index.csv

+filename,sep,skip,contents
+BCITRAITS_20101220.csv,c,0, 
+census1/PlotsDataReport.txt,t,0, 
+TaxonomyDataReport.txt,t,0, 

ms/metadata/BCI/_site_details.csv

+Item,Details
+contact name,Joe Wright
+contact email,wrightj@si.edu
+contact affiliation, "Centre for Tropical Science"
+dataset name, "Barro colorado island, 50 ha plot"
+country, Panama
+climate, Tropical
+vegetation type, Tropical rain forest
+minimum tree size, 1cm dbh
+field methodology, "All free-standing woody plants with diameter at breast height (dbh) > 1 cm were measured in a 50-ha plot on BCI in 1982, 1985, 1990, 1995, 2000, and 2005."
+trait information,
+key references, "Hubbell and Foster 1992, Condit et al. 2006"

ms/metadata/BCI/files/BCITRAITS_20101220.csv

+use,var,units,description
1,GENUS,,genus name truncated at 12 letters
1,SPECIES,,species name truncated at 12 letters
1,FAMILY,,family name truncated at 12 letters
1,GRWFRM2,,Values are as in GRWFRM1 except free-standing species can have multiple values if maximum size varies widely within Panama 
1,GRWFRM1,,"Values are Climber, HERB, S, U, M and T. S, U, M and T are free-standing species with maximum heights of 5, 10, 20 and > 30 m, respectively. "
1,SP,,six letter species code 
1,SG60C_AVG,g/cm3,mean wood specific gravity after drying at 60C (g cm-3)
1,SG100C_AVG,g/cm3,mean wood specific gravity after drying at 100C (g cm-3)
1,SG60C_SEM,g/cm3,standard error of SG60C_AVG (g cm-3)
1,SG100C_SEM,g/cm3,standard error of SG100C_AVG (g cm-3)
1,SG60C_N,,number of individuals sampled for SG60C_AVG
1,SG100C_N,,number of individuals sampled for SG100C_AVG
1,SEED_DRY,g,"mean seed dry mass after drying at 60 C (g),  where seed is defined to include the endosperm and embryo only"
1,DBH_AVG,mm,mean DBH measured in 2005 of up to the six largest individuals in the BCI 50-ha plot (mm)
1,HEIGHT_AVG,m,mean height of up to the six largest individuals in the BCI 50-ha plot (m)
1,DIAM_AVG,m,mean crown diameter of up to the six largest individuals in the BCI 50-ha plot (m)
1,DBH_SEM,mm,one standard error of DBH_AVG (mm)
1,HEIGHT_SEM,m,one standard error of HEIGHT_AVG (m)
1,DIAM_SEM,m,one standard error of DIAM_AVG (m)
1,LMALAM_AVD,g/m2,mean leaf mass per unit area measured for the leaf lamina excluding the petiole and for compound leaves the petiolules (g m-2) for leaves receiving direct sunlight
1,LMALAM_SED,g/m2,one standard error for LMALAM_AVD (g m-2)
\ No newline at end of file

ms/metadata/BCI/files/TaxonomyDataReport.csv

+use,var,units,description
+0,Family,,
+0,Genus,,
+0,species,,
+0,subspecies,,
+0,mnemonic,,
+0,IDlevel,,
+0,Authority,,
+0,PriorNames,,
+0,SpeciesID,,

ms/metadata/BCI/files/census1_PlotsDataReport.csv

+use,var,units,description
+0,Plot,,
+0,Latin,,
+0,Quadrat,,
+0,gx,,
+0,gy,,
+0,TreeID,,
+0,Tag,,
+0,StemID,,
+0,StemTag,,
+0,Census,,
+0,DBH,,
+0,HOM,,
+0,Date,,
+0,Codes,,
+0,Stem,,
+0,Status,,

ms/metadata/Canada/_file_index.csv

+filename,sep,skip,contents
+Canada_Data2George_20130818.csv,c,0, 
+EcoregionCodes.csv,c,0, 
+FIA_REF_SPECIES.csv,c,0, 

ms/metadata/Canada/_site_details.csv

+Item,Details
+contact name,
+contact email,
+contact affiliation,
+dataset name,
+country,
+climate,
+vegetation type,
+minimum tree size,
+field methodology,
+trait information,
+key references,

ms/metadata/Canada/files/Canada_Data2George_20130818.csv

+use,var,units,description
+0,PLOTTREE_I,,
+0,Species_FIAcode,,
+0,InitDBH,,
+0,FinalDBH,,
+0,PLOT_ID,,
+0,Subplot_ID,,
+0,SubPlot_Size,,
+0,Lat,,
+0,Lon,,
+0,IndWeight,,
+0,IntervalYears,,
+0,Ecocode,,
+0,MAT,,
+0,MAP,,
+0,Province,,

ms/metadata/Canada/files/EcoregionCodes.csv

+use,var,units,description
+0,ECOCODE,,
+0,DOMAIN,,
+0,DIVISION,,
+0,PROVINCE,,

ms/metadata/Canada/files/FIA_REF_SPECIES.csv

+use,var,units,description
+0,SPCD,,
+0,COMMON_NAME,,
+0,GENUS,,
+0,SPECIES,,
+0,VARIETY,,
+0,SUBSPECIES,,
+0,SPECIES_SYMBOL,,
+0,E_SPGRPCD,,
+0,W_SPGRPCD,,
+0,C_SPGRPCD,,
+0,P_SPGRPCD,,
+0,MAJOR_SPGRPCD,,
+0,STOCKING_SPGRPCD,,
+0,FOREST_TYPE_SPGRPCD,,
+0,EXISTS_IN_NCRS,,
+0,EXISTS_IN_NERS,,
+0,EXISTS_IN_PNWRS,,
+0,EXISTS_IN_RMRS,,
+0,EXISTS_IN_SRS,,
+0,SITETREE,,
+0,SFTWD_HRDWD,,
+0,ST_EXISTS_IN_NCRS,,
+0,ST_EXISTS_IN_NERS,,
+0,ST_EXISTS_IN_PNWRS,,
+0,ST_EXISTS_IN_RMRS,,
+0,ST_EXISTS_IN_SRS,,
+0,CORE,,
+0,EAST,,
+0,WEST,,
+0,CARIBBEAN,,
+0,PACIFIC,,
+0,WOODLAND,,
+0,MANUAL_START,,
+0,MANUAL_END,,
+0,JENKINS_SPGRPCD,,
+0,JENKINS_TOTAL_B1,,
+0,JENKINS_TOTAL_B2,,
+0,JENKINS_STEM_WOOD_RATIO_B1,,
+0,JENKINS_STEM_WOOD_RATIO_B2,,
+0,JENKINS_STEM_BARK_RATIO_B1,,
+0,JENKINS_STEM_BARK_RATIO_B2,,
+0,JENKINS_FOLIAGE_RATIO_B1,,
+0,JENKINS_FOLIAGE_RATIO_B2,,
+0,JENKINS_ROOT_RATIO_B1,,
+0,JENKINS_ROOT_RATIO_B2,,
+0,JENKINS_SAPLING_ADJUSTMENT,,
+0,WOOD_SPGR_GREENVOL_DRYWT,,
+0,WOOD_SPGR_GREENVOL_DRYWT_CIT,,
+0,BARK_SPGR_GREENVOL_DRYWT,,
+0,BARK_SPGR_GREENVOL_DRYWT_CIT,,
+0,MC_PCT_GREEN_BARK,,
+0,MC_PCT_GREEN_BARK_CIT,,
+0,MC_PCT_GREEN_WOOD,,
+0,MC_PCT_GREEN_WOOD_CIT,,
+0,WOOD_SPGR_MC12VOL_DRYWT,,
+0,WOOD_SPGR_MC12VOL_DRYWT_CIT,,
+0,BARK_VOL_PCT,,
+0,BARK_VOL_PCT_CIT,,
+0,RAILE_STUMP_DOB_B1,,
+0,RAILE_STUMP_DIB_B1,,
+0,RAILE_STUMP_DIB_B2,,
+0,CWD_DECAY_RATIO1,,
+0,CWD_DECAY_RATIO2,,
+0,CWD_DECAY_RATIO3,,
+0,CWD_DECAY_RATIO4,,
+0,CWD_DECAY_RATIO5,,
+0,DWM_CARBON_RATIO,,
+0,STANDING_DEAD_DECAY_RATIO1,,
+0,STANDING_DEAD_DECAY_RATIO2,,
+0,STANDING_DEAD_DECAY_RATIO3,,
+0,STANDING_DEAD_DECAY_RATIO4,,
+0,STANDING_DEAD_DECAY_RATIO5,,
+0,CREATED_BY,,
+0,CREATED_DATE,,
+0,CREATED_IN_INSTANCE,,
+0,MODIFIED_BY,,
+0,MODIFIED_DATE,,
+0,MODIFIED_IN_INSTANCE,,

ms/metadata/France/_file_index.csv

+filename,sep,skip,contents
+2005/arbres_foret_2005.csv,c,0, 
+2005/arbres_morts_foret_2005.csv,c,0, 
+2005/arbres_peupleraie_2005.csv,c,0, 
+2005/documentation_2005-2.csv,c,0, 
+2005/documentation_flore.csv,c,0, 
+2005/ecologie_2005.csv,c,0, 
+2005/flore_2005.csv,c,0, 
+2005/placettes_foret_2005.csv,c,0, 
+2005/placettes_peupleraie_2005.csv,c,0, 
+2006/arbres_foret_2006.csv,c,0, 
+2006/arbres_morts_foret_2006.csv,c,0, 
+2006/arbres_peupleraie_2006.csv,c,0, 
+2006/couverts_foret_2006.csv,c,0, 
+2006/documentation_2006-2.csv,c,0, 
+2006/documentation_flore.csv,c,0, 
+2006/ecologie_2006.csv,c,0, 
+2006/flore_2006.csv,c,0, 
+2006/placettes_foret_2006.csv,c,0, 
+2006/placettes_peupleraie_2006.csv,c,0, 
+2007/arbres_foret_2007.csv,c,0, 
+2007/arbres_morts_foret_2007.csv,c,0, 
+2007/arbres_peupleraie_2007.csv,c,0, 
+2007/couverts_foret_2007.csv,c,0, 
+2007/documentation_2007-2.csv,c,0, 
+2007/documentation_flore.csv,c,0, 
+2007/ecologie_2007.csv,c,0, 
+2007/flore_2007.csv,c,0, 
+2007/placettes_foret_2007.csv,c,0, 
+2007/placettes_peupleraie_2007.csv,c,0, 
+2008/arbres_foret_2008.csv,c,0, 
+2008/arbres_morts_foret_2008.csv,c,0, 
+2008/arbres_morts_peupleraie_2008.csv,c,0, 
+2008/arbres_peupleraie_2008.csv,c,0, 
+2008/couverts_foret_2008.csv,c,0, 
+2008/documentation_2008-2.csv,c,0, 
+2008/documentation_flore.csv,c,0, 
+2008/ecologie_2008.csv,c,0, 
+2008/flore_2008.csv,c,0, 
+2008/placettes_foret_2008.csv,c,0, 
+2008/placettes_peupleraie_2008.csv,c,0, 
+2009/arbres_foret_2009.csv,c,0, 
+2009/arbres_morts_foret_2009.csv,c,0, 
+2009/arbres_morts_peupleraie_2009.csv,c,0, 
+2009/arbres_peupleraie_2009.csv,c,0, 
+2009/couverts_foret_2009.csv,c,0, 
+2009/documentation_2009-2.csv,c,0, 
+2009/documentation_flore.csv,c,0, 
+2009/ecologie_2009.csv,c,0, 
+2009/flore_2009.csv,c,0, 
+2009/placettes_foret_2009.csv,c,0, 
+2009/placettes_peupleraie_2009.csv,c,0, 
+2010/arbres_foret_2010.csv,c,0, 
+2010/arbres_morts_foret_2010.csv,c,0, 
+2010/arbres_morts_peupleraie_2010.csv,c,0, 
+2010/arbres_peupleraie_2010.csv,c,0, 
+2010/couverts_foret_2010.csv,c,0, 
+2010/documentation_2010.csv,c,0, 
+2010/documentation_flore.csv,c,0, 
+2010/ecologie_2010.csv,c,0, 
+2010/flore_2010.csv,c,0, 
+2010/placettes_foret_2010.csv,c,0, 
+2010/placettes_peupleraie_2010.csv,c,0, 
+2011/arbres_foret_2011.csv,c,0, 
+2011/arbres_morts_foret_2011.csv,c,0, 
+2011/arbres_morts_peupleraie_2011.csv,c,0, 
+2011/arbres_peupleraie_2011.csv,c,0, 
+2011/couverts_foret_2011.csv,c,0, 
+2011/documentation_2011.csv,c,0, 
+2011/documentation_flore.csv,c,0, 
+2011/ecologie_2011.csv,c,0, 
+2011/flore_2011.csv,c,0, 
+2011/placettes_foret_2011.csv,c,0, 
+2011/placettes_peupleraie_2011.csv,c,0, 
+altitude/SER_alti_2011.csv,c,0, 
+altitude/SER_alti.csv,c,0, 
+climate_piedallu/placettesGK_avec 2011.csv,c,0, 
+climate_piedallu/placettesGK2.csv,c,0, 
+climate_piedallu/texture.txt,t,0, 
+cycle3/arbres2.txt,t,0, 
+cycle3/data.arbre.tot.txt,t,0, 
+cycle3/speciesnames.txt,t,0, 

ms/metadata/France/_site_details.csv

+Item,Details
+contact name,
+contact email,
+contact affiliation,
+dataset name,
+country,
+climate,
+vegetation type,
+minimum tree size,
+field methodology,
+trait information,
+key references,

ms/metadata/France/files/2005_arbres_foret_2005.csv

+use,var,units,description
+0,idp.a.veget.espar.ori.lib.forme.tige.c13.ir5.htot.q1.q2.q3.r.lfsd.v.w,,

ms/metadata/France/files/2005_arbres_morts_foret_2005.csv

+use,var,units,description
+0,idp.a.espar.ori.veget.c0.v.w,,

ms/metadata/France/files/2005_arbres_peupleraie_2005.csv

+use,var,units,description
+0,idp.a.veget.espar.clon.ori.c13.htot.q1.q2.q3.r.lfsd.v.w,,

ms/metadata/France/files/2005_documentation_2005-2.csv

+use,var,units,description
+0,donnee.code.libelle,,

ms/metadata/France/files/2005_documentation_flore.csv

+use,var,units,description
+0,codtax.nomlatin.lib_ref.numnomen,,

ms/metadata/France/files/2005_ecologie_2005.csv

+use,var,units,description
+0,idp.dateeco.obsdate.topo.obstopo.pent2.expo.masque.humus.obspedo.roche.obsroc.affroc.afpla.cailloux.cai40.text2.text1.prof2.prof1.obsprof.pcalc.pcalf.pox.ppseudo.pgley.obshydr.tsol.lign1.lign2.herb.obsveget,,