From f709c0fac1420698f95a80775690d7f3230d67de Mon Sep 17 00:00:00 2001
From: Rousseau Vincent <vincent.rousseau@irstea.fr>
Date: Tue, 5 Mar 2019 11:49:04 +0100
Subject: [PATCH] Add influx to csv

---
 docker-compose.yml        | 20 +++++++++++++++--
 influxdb/Dockerfile       | 10 +++++++++
 influxdb/influx_to_csv.py | 45 +++++++++++++++++++++++++++++++++++++++
 influxdb/requirements.txt |  1 +
 4 files changed, 74 insertions(+), 2 deletions(-)
 create mode 100644 influxdb/Dockerfile
 create mode 100644 influxdb/influx_to_csv.py
 create mode 100644 influxdb/requirements.txt

diff --git a/docker-compose.yml b/docker-compose.yml
index 1284f27..1922b00 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,20 +1,36 @@
 version: '3'
 volumes:
-  persitent-storage:
+  influxdb-storage:
+  grafana-storage:
 services:
   influxdb:
     image: influxdb:1.7
     restart: always
+    ports:
+      - "8086:8086"
+    volumes:
+      - 'influxdb-storage:/var/lib/influxdb'
   grafana:
     image: grafana/grafana:5.4.3
     restart: always
     ports:
       - "3000:3000"
     volumes:
-      - 'persitent-storage:/var/lib/grafana'
+      - 'grafana-storage:/var/lib/grafana'
       - ./grafana/provisioning/:/etc/grafana/provisioning/
     env_file:
       - ./grafana/config.monitoring
+  influx_to_csv:
+    build: ./influxdb
+    restart: always
+    volumes:
+      - ./data/:/var/lib/csv/
+    environment:
+      - INFLUX_HOST=influxdb
+      - CSV_FOLDER_PATH=/var/lib/csv/
+    depends_on:
+      - influxdb
+      - grafana
   cpu_load:
     build: ./sensors/cpu_load
     restart: always
diff --git a/influxdb/Dockerfile b/influxdb/Dockerfile
new file mode 100644
index 0000000..df44a82
--- /dev/null
+++ b/influxdb/Dockerfile
@@ -0,0 +1,10 @@
+FROM python:3
+
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+
+COPY requirements.txt /usr/src/app
+RUN python3 -m pip install -r requirements.txt
+
+COPY influx_to_csv.py /usr/src/app
+CMD ["python3", "/usr/src/app/influx_to_csv.py"]
\ No newline at end of file
diff --git a/influxdb/influx_to_csv.py b/influxdb/influx_to_csv.py
new file mode 100644
index 0000000..e8a6067
--- /dev/null
+++ b/influxdb/influx_to_csv.py
@@ -0,0 +1,45 @@
+import time
+import os
+from datetime import datetime, timedelta
+from pathlib import Path
+import csv
+
+from influxdb import InfluxDBClient
+
+influx_host = os.getenv("INFLUX_HOST", "localhost")
+influx_client = InfluxDBClient(host=influx_host, database="cpu-sense")
+influx_client.create_database("cpu-sense")
+
+write_time_interval = 20
+start_time = datetime.utcnow()
+begin = start_time - timedelta(seconds=100)
+end = begin + timedelta(seconds=write_time_interval)
+
+csv_folder_path = os.getenv("CSV_FOLDER_PATH", "./data")
+data_folder = Path(csv_folder_path)
+data_folder.mkdir(parents=True, exist_ok=True)
+
+while True:
+    begin += timedelta(seconds=write_time_interval)
+    end += timedelta(seconds=write_time_interval)
+    query = 'select * from "CPU Usage" where time > \'' + begin.isoformat() + "Z' AND time <= '" + end.isoformat() + "Z' tz('Europe/Paris')"
+    result = influx_client.query(query, epoch="ms")
+
+    try:
+        n = next(result.get_points())
+    except StopIteration:
+        print('Influxdb result query is empty')
+    else:
+        file_date = datetime.now().strftime('%Y_%m_%d_%H_%M')
+        file_name = data_folder/str('cpu_data_' + file_date + '.csv')
+        with file_name.open('a') as csvfile:
+            fieldnames = list(next(result.get_points()).keys())
+            writer = csv.DictWriter(csvfile, fieldnames)
+            if file_name.stat().st_size == 0:
+                writer.writeheader()
+            for row in result.get_points():
+                # query timestamp from influxdb in ms precision
+                row['time'] /= 1000.0
+                writer.writerow(row)
+
+    time.sleep(write_time_interval - (time.time() - start_time.timestamp()) % write_time_interval)
diff --git a/influxdb/requirements.txt b/influxdb/requirements.txt
new file mode 100644
index 0000000..9c46347
--- /dev/null
+++ b/influxdb/requirements.txt
@@ -0,0 +1 @@
+influxdb==5.2.1
-- 
GitLab