script R added and tested

input/datasources.csv

-id;datasetName;datasetProvider;datasetURL;geoExent;temporalExent;semantic
-1;All-plu;3M;https://data.montpellier3m.fr/dataset/plu-des-communes-de-montpellier-mediterranee-metropole;Montpellier Méditerranée métropole;2019;"""urban planning, city, land cover"""
-2;Montpellier-plu;3M;https://data.montpellier3m.fr/dataset/plan-local-durbanisme-de-montpellier;Montpellier;2019;"""urban planning, city, land cover"""
-3;Land-cover;3M;https://data.montpellier3m.fr/dataset/evolution-de-loccupation-du-sol-de-montpellier-mediterranee-metropole;Montpellier Méditerranée métropole;1994-2017;"""land cover, mapping"""
-4;Population-distribution;3M;http://data.montpellier3m.fr/dataset/distribution-fine-de-la-population-2013-au-bati;Montpellier;2013;"""population"""
-5;Resident-Mobility;3M;http://data.montpellier3m.fr/dataset/enquete-menages-deplacements;Montpellier;2015;"""mobility, transport, population, car"""
-6;Car-counting;3M;http://data.montpellier3m.fr/dataset/comptage-vehicules-particuliers-de-montpellier;Montpellier;2011-2019;"""mobility, transport"""
-7;Car-way;3M;http://data.montpellier3m.fr/dataset/pdu-hierarchisation-des-voies-de-la-metropole;Montpellier Méditerranée métropole;2018;"""car ways"""
-9;Transport-offer;3M;http://data.montpellier3m.fr/dataset/offre-de-transport-tam-en-gtfs;Montpellier Méditerranée métropole;2019;"""mobility, bus, tramway"""
+"id";"datasetName";"datasetProvider";"datasetURL";"geoExent";"temporalExtent";"semantic";"UUID"
+1;"All-plu";"3M";"https://data.montpellier3m.fr/dataset/plu-des-communes-de-montpellier-mediterranee-metropole";"Montpellier Méditerranée Métropole";2019;"urban planning, city, land cover";"8b69f88c-224a-11ea-978f-2e728ce88125"
+2;"Montpellier-plu";"3M";"https://data.montpellier3m.fr/dataset/plan-local-durbanisme-de-montpellier";"Montpellier";2019;"urban planning, city, land cover";"8b69fbb6-224a-11ea-978f-2e728ce88125"
+3;"Land-cover";"3M";"https://data.montpellier3m.fr/dataset/evolution-de-loccupation-du-sol-de-montpellier-mediterranee-metropole";"Montpellier Méditerranée Métropole";"1994-2017";"land cover, mapping";"8b69fd28-224a-11ea-978f-2e728ce88125"
+4;"Population-distribution";"3M";"http://data.montpellier3m.fr/dataset/distribution-fine-de-la-population-2013-au-bati";"Montpellier";2013;"population";"8b6a00a2-224a-11ea-978f-2e728ce88125"
+5;"Resident-Mobility";"3M";"http://data.montpellier3m.fr/dataset/enquete-menages-deplacements";"Montpellier";2015;"mobility, transport, population, car";"8b6a01f6-224a-11ea-978f-2e728ce88125"
+6;"Car-counting";"3M";"http://data.montpellier3m.fr/dataset/comptage-vehicules-particuliers-de-montpellier";"Montpellier";"2011-2019";"mobility, transport";"8b6a0336-224a-11ea-978f-2e728ce88125"
+7;"Car-way";"3M";"http://data.montpellier3m.fr/dataset/pdu-hierarchisation-des-voies-de-la-metropole";"Montpellier Méditerranée Métropole";2018;"car ways";"8b6a046c-224a-11ea-978f-2e728ce88125"
+9;"Transport-offer";"3M";"http://data.montpellier3m.fr/dataset/offre-de-transport-tam-en-gtfs";"Montpellier Méditerranée Métropole";2019;"mobility, bus, tramway";"8b6a05b6-224a-11ea-978f-2e728ce88125"

output/data/.forgit

+to track the tree for git repository excluding data files versionning
\ No newline at end of file

output/meta/.forgit

+to track the tree for git repository excluding data files versionning
\ No newline at end of file

src/addServicesToGN.R

+library(geonapi)
+library(geometa)
+library(uuid)
+library(osmdata)
+
+working_dir = getwd()
+
+## Connection to geonetwork
+gn <- GNManager$new(
+  url = "http://10.0.0.9:8080/geonetwork",
+  version = "3.6.1",
+  user = "admin",
+  pwd = "admin"
+  # logger = "DEBUG"
+)
+
+## Read input
+services <- read.csv(file=paste(working_dir, "../input/datasources.csv", sep = "/"), sep =";")
+
+# Browse datasources and create MD
+for (service in services$id) {
+  print(paste0("Working on: ", services$datasetName[service]))
+  metadata_id <- services$uuid[service]
+  # For temporal extent
+  ISOMetadataNamespace[["GML"]]$uri <- "http://www.opengis.net/gml"
+  
+  ## MD creation
+  md = ISOMetadata$new()
+  metadata_id=paste(metadata_id)
+  md$setFileIdentifier(metadata_id)
+  md$setCharacterSet("utf8")
+  md$setMetadataStandardName("ISO 19115:2003/19139")
+  md$setLanguage("eng")
+  md$setDateStamp(Sys.time())
+  
+  ## identification
+  ident <- ISODataIdentification$new()
+  ident$setAbstract(paste(services$datasetName[service]))
+  ident$setLanguage("fra")
+  # for (topic in unlist(strsplit(paste(services$topic[service]), ", "))){
+  #   ident$addTopicCategory(topic)
+  # }
+  
+  ## keywords
+  ### General Keywords
+  dynamic_keywords <- ISOKeywords$new()
+  for (kw in unlist(strsplit(noquote(paste(services$semantic[service])), ", "))){
+    dynamic_keywords$addKeyword(kw)
+  }
+  ident$addKeywords(dynamic_keywords)
+  
+    # #add link to data access
+    # distrib <- ISODistribution$new()
+    # dto <- ISODigitalTransferOptions$new()
+    # for (link in unlist(strsplit(paste(services$web.access[service]), ", "))){
+    #   # Remove paranthesis
+    #   tuple <- gsub('\\(',"",link)
+    #   tuple <- gsub('\\)',"",tuple)
+    #   newURL <- ISOOnlineResource$new()
+    #   newURL$setName(paste0(strsplit(paste(tuple), " @ ")[[1]][1]," :"))
+    #   newURL$setLinkage(strsplit(paste(tuple), " @ ")[[1]][2])
+    #   newURL$setProtocol("WWW:LINK-1.0-http--link")
+    #   dto$addOnlineResource(newURL)
+    # }
+    # distrib$setDigitalTransferOptions(dto)
+    # md$setDistributionInfo(distrib)
+    
+    # Title and identification
+    ct <- ISOCitation$new()
+    ct$setTitle(paste(services$datasetName[service]))
+    isoid=ISOMetaIdentifier$new(code = services$uuid[service])
+    ct$addIdentifier(isoid)
+    ident$setCitation(ct)
+    ## Aperçu / thumbnail
+    # for(thumbnail in unlist(strsplit(paste(services$thumbnail[service]), ", "))){
+    #   go <- ISOBrowseGraphic$new(
+    #     fileName = thumbnail,
+    #     fileDescription = "thumbnail",
+    #     fileType = "image/png"
+    #   )
+    #   ident$addGraphicOverview(go)
+    # }
+    
+    # Temporal Extent
+    te <- ISOTemporalExtent$new()
+    if (grepl("-", paste(services$temporalExtent[service]) )){
+      # start <- ISOdate(paste(services$temporalExtent[service]), 1, 1, 0, 0, 1)
+      # end <- ISOdate(paste(services$temporalExtent[service]), 12, 31, 23, 59, 59)
+      # tp <- GMLTimePeriod$new(beginPosition = start, endPosition = end)
+    }
+    else {
+      start <- ISOdate(paste(services$temporalExtent[service]), 1, 1, 0, 0, 1)
+      end <- ISOdate(paste(services$temporalExtent[service]), 12, 31, 23, 59, 59)
+      tp <- GMLTimePeriod$new(beginPosition = start, endPosition = end)
+    }
+    te$setTimePeriod(tp)
+    extent <- ISOExtent$new()
+    extent$setTemporalElement(te)
+    ident$addExtent(extent)
+    
+    #Spatial  extent
+    extent <- ISOExtent$new()
+    bb = getbb(place_name=services$geoExent[service], featuretype = "boundary")
+    spatialExtent <- ISOGeographicBoundingBox$new(minx = bb[1,1], miny = bb[2,1], maxx = bb[1,2], maxy = bb[2,2])
+    extent$setGeographicElement(spatialExtent)
+    ident$addExtent(extent)
+    
+    md$addIdentificationInfo(ident)
+
+    
+    ## Insert or update
+    # An update has to be done based on the internal Geonetwork id (that can be queried as well)
+    created = gn$insertMetadata(
+      xml = md$encode(),
+      group = "1",
+      category = "dataset"
+    )
+}

src/main.py

@@ -46,15 +46,18 @@ def getUrlFromOpendata3M(inputCSV):
 
      # Step 4
     opendata3mDataMetada = {}
+    idcsv = 1
     for node in idNodeList:
         opendata3mData = []
-        nodeDataMetada = {'metadata': None, 'data': None}
+        nodeDataMetada = {'metadata': None, 'data': None, 'idCSV': idcsv}
         metadata = requests.get("http://data.montpellier3m.fr/api/3/action/package_show?id="+node[0]).json()
         #get resources
         for resource in metadata['result']['resources']:
             opendata3mData.append(resource['url'])
         nodeDataMetada['data'] = opendata3mData
+        nodeDataMetada['metadata'] = metadata
         opendata3mDataMetada.update({str(node): nodeDataMetada})
+        idcsv = idcsv+1
 
     return opendata3mDataMetada
 
@@ -94,5 +97,9 @@ if __name__ == '__main__':
     """Download File"""
     nboffiledl = downloadOpendata3MFiles(opendata3mDataMetada, pathToSaveDownloadedData)
 
+    """Insert files inside HDFS"""
+
+    """Build and insert iso19139 xml to geonetwork"""
+
     print(str(nboffiledl)+" files downloaded in : " + pathToSaveDownloadedData)
     print("AIDMOIt ingestion module ends")