Commit 8ddef70c authored by Decoupes Remy's avatar Decoupes Remy
Browse files

script R added and tested

No related merge requests found
Showing with 137 additions and 10 deletions
+137 -10
id;datasetName;datasetProvider;datasetURL;geoExent;temporalExent;semantic
1;All-plu;3M;https://data.montpellier3m.fr/dataset/plu-des-communes-de-montpellier-mediterranee-metropole;Montpellier Méditerranée métropole;2019;"""urban planning, city, land cover"""
2;Montpellier-plu;3M;https://data.montpellier3m.fr/dataset/plan-local-durbanisme-de-montpellier;Montpellier;2019;"""urban planning, city, land cover"""
3;Land-cover;3M;https://data.montpellier3m.fr/dataset/evolution-de-loccupation-du-sol-de-montpellier-mediterranee-metropole;Montpellier Méditerranée métropole;1994-2017;"""land cover, mapping"""
4;Population-distribution;3M;http://data.montpellier3m.fr/dataset/distribution-fine-de-la-population-2013-au-bati;Montpellier;2013;"""population"""
5;Resident-Mobility;3M;http://data.montpellier3m.fr/dataset/enquete-menages-deplacements;Montpellier;2015;"""mobility, transport, population, car"""
6;Car-counting;3M;http://data.montpellier3m.fr/dataset/comptage-vehicules-particuliers-de-montpellier;Montpellier;2011-2019;"""mobility, transport"""
7;Car-way;3M;http://data.montpellier3m.fr/dataset/pdu-hierarchisation-des-voies-de-la-metropole;Montpellier Méditerranée métropole;2018;"""car ways"""
9;Transport-offer;3M;http://data.montpellier3m.fr/dataset/offre-de-transport-tam-en-gtfs;Montpellier Méditerranée métropole;2019;"""mobility, bus, tramway"""
"id";"datasetName";"datasetProvider";"datasetURL";"geoExent";"temporalExtent";"semantic";"UUID"
1;"All-plu";"3M";"https://data.montpellier3m.fr/dataset/plu-des-communes-de-montpellier-mediterranee-metropole";"Montpellier Méditerranée Métropole";2019;"urban planning, city, land cover";"8b69f88c-224a-11ea-978f-2e728ce88125"
2;"Montpellier-plu";"3M";"https://data.montpellier3m.fr/dataset/plan-local-durbanisme-de-montpellier";"Montpellier";2019;"urban planning, city, land cover";"8b69fbb6-224a-11ea-978f-2e728ce88125"
3;"Land-cover";"3M";"https://data.montpellier3m.fr/dataset/evolution-de-loccupation-du-sol-de-montpellier-mediterranee-metropole";"Montpellier Méditerranée Métropole";"1994-2017";"land cover, mapping";"8b69fd28-224a-11ea-978f-2e728ce88125"
4;"Population-distribution";"3M";"http://data.montpellier3m.fr/dataset/distribution-fine-de-la-population-2013-au-bati";"Montpellier";2013;"population";"8b6a00a2-224a-11ea-978f-2e728ce88125"
5;"Resident-Mobility";"3M";"http://data.montpellier3m.fr/dataset/enquete-menages-deplacements";"Montpellier";2015;"mobility, transport, population, car";"8b6a01f6-224a-11ea-978f-2e728ce88125"
6;"Car-counting";"3M";"http://data.montpellier3m.fr/dataset/comptage-vehicules-particuliers-de-montpellier";"Montpellier";"2011-2019";"mobility, transport";"8b6a0336-224a-11ea-978f-2e728ce88125"
7;"Car-way";"3M";"http://data.montpellier3m.fr/dataset/pdu-hierarchisation-des-voies-de-la-metropole";"Montpellier Méditerranée Métropole";2018;"car ways";"8b6a046c-224a-11ea-978f-2e728ce88125"
9;"Transport-offer";"3M";"http://data.montpellier3m.fr/dataset/offre-de-transport-tam-en-gtfs";"Montpellier Méditerranée Métropole";2019;"mobility, bus, tramway";"8b6a05b6-224a-11ea-978f-2e728ce88125"
to track the tree for git repository excluding data files versionning
\ No newline at end of file
to track the tree for git repository excluding data files versionning
\ No newline at end of file
library(geonapi)
library(geometa)
library(uuid)
library(osmdata)
working_dir = getwd()
## Connection to geonetwork
gn <- GNManager$new(
url = "http://10.0.0.9:8080/geonetwork",
version = "3.6.1",
user = "admin",
pwd = "admin"
# logger = "DEBUG"
)
## Read input
services <- read.csv(file=paste(working_dir, "../input/datasources.csv", sep = "/"), sep =";")
# Browse datasources and create MD
for (service in services$id) {
print(paste0("Working on: ", services$datasetName[service]))
metadata_id <- services$uuid[service]
# For temporal extent
ISOMetadataNamespace[["GML"]]$uri <- "http://www.opengis.net/gml"
## MD creation
md = ISOMetadata$new()
metadata_id=paste(metadata_id)
md$setFileIdentifier(metadata_id)
md$setCharacterSet("utf8")
md$setMetadataStandardName("ISO 19115:2003/19139")
md$setLanguage("eng")
md$setDateStamp(Sys.time())
## identification
ident <- ISODataIdentification$new()
ident$setAbstract(paste(services$datasetName[service]))
ident$setLanguage("fra")
# for (topic in unlist(strsplit(paste(services$topic[service]), ", "))){
# ident$addTopicCategory(topic)
# }
## keywords
### General Keywords
dynamic_keywords <- ISOKeywords$new()
for (kw in unlist(strsplit(noquote(paste(services$semantic[service])), ", "))){
dynamic_keywords$addKeyword(kw)
}
ident$addKeywords(dynamic_keywords)
# #add link to data access
# distrib <- ISODistribution$new()
# dto <- ISODigitalTransferOptions$new()
# for (link in unlist(strsplit(paste(services$web.access[service]), ", "))){
# # Remove paranthesis
# tuple <- gsub('\\(',"",link)
# tuple <- gsub('\\)',"",tuple)
# newURL <- ISOOnlineResource$new()
# newURL$setName(paste0(strsplit(paste(tuple), " @ ")[[1]][1]," :"))
# newURL$setLinkage(strsplit(paste(tuple), " @ ")[[1]][2])
# newURL$setProtocol("WWW:LINK-1.0-http--link")
# dto$addOnlineResource(newURL)
# }
# distrib$setDigitalTransferOptions(dto)
# md$setDistributionInfo(distrib)
# Title and identification
ct <- ISOCitation$new()
ct$setTitle(paste(services$datasetName[service]))
isoid=ISOMetaIdentifier$new(code = services$uuid[service])
ct$addIdentifier(isoid)
ident$setCitation(ct)
## Aperçu / thumbnail
# for(thumbnail in unlist(strsplit(paste(services$thumbnail[service]), ", "))){
# go <- ISOBrowseGraphic$new(
# fileName = thumbnail,
# fileDescription = "thumbnail",
# fileType = "image/png"
# )
# ident$addGraphicOverview(go)
# }
# Temporal Extent
te <- ISOTemporalExtent$new()
if (grepl("-", paste(services$temporalExtent[service]) )){
# start <- ISOdate(paste(services$temporalExtent[service]), 1, 1, 0, 0, 1)
# end <- ISOdate(paste(services$temporalExtent[service]), 12, 31, 23, 59, 59)
# tp <- GMLTimePeriod$new(beginPosition = start, endPosition = end)
}
else {
start <- ISOdate(paste(services$temporalExtent[service]), 1, 1, 0, 0, 1)
end <- ISOdate(paste(services$temporalExtent[service]), 12, 31, 23, 59, 59)
tp <- GMLTimePeriod$new(beginPosition = start, endPosition = end)
}
te$setTimePeriod(tp)
extent <- ISOExtent$new()
extent$setTemporalElement(te)
ident$addExtent(extent)
#Spatial extent
extent <- ISOExtent$new()
bb = getbb(place_name=services$geoExent[service], featuretype = "boundary")
spatialExtent <- ISOGeographicBoundingBox$new(minx = bb[1,1], miny = bb[2,1], maxx = bb[1,2], maxy = bb[2,2])
extent$setGeographicElement(spatialExtent)
ident$addExtent(extent)
md$addIdentificationInfo(ident)
## Insert or update
# An update has to be done based on the internal Geonetwork id (that can be queried as well)
created = gn$insertMetadata(
xml = md$encode(),
group = "1",
category = "dataset"
)
}
......@@ -46,15 +46,18 @@ def getUrlFromOpendata3M(inputCSV):
# Step 4
opendata3mDataMetada = {}
idcsv = 1
for node in idNodeList:
opendata3mData = []
nodeDataMetada = {'metadata': None, 'data': None}
nodeDataMetada = {'metadata': None, 'data': None, 'idCSV': idcsv}
metadata = requests.get("http://data.montpellier3m.fr/api/3/action/package_show?id="+node[0]).json()
#get resources
for resource in metadata['result']['resources']:
opendata3mData.append(resource['url'])
nodeDataMetada['data'] = opendata3mData
nodeDataMetada['metadata'] = metadata
opendata3mDataMetada.update({str(node): nodeDataMetada})
idcsv = idcsv+1
return opendata3mDataMetada
......@@ -94,5 +97,9 @@ if __name__ == '__main__':
"""Download File"""
nboffiledl = downloadOpendata3MFiles(opendata3mDataMetada, pathToSaveDownloadedData)
"""Insert files inside HDFS"""
"""Build and insert iso19139 xml to geonetwork"""
print(str(nboffiledl)+" files downloaded in : " + pathToSaveDownloadedData)
print("AIDMOIt ingestion module ends")
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment