Updates requirements for mqtt; adds setups for mqtt and logging

LICENSE

-                    GNU GENERAL PUBLIC LICENSE
+LICENSE                    GNU GENERAL PUBLIC LICENSE
                        Version 3, 29 June 2007
 
  Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>

compressed_sized_timed_rotating_logger.py

+import time
+import os
+import logging.handlers as handlers
+import zipfile
+
+
+class CompressedSizedTimedRotatingFileHandler(handlers.TimedRotatingFileHandler):
+    """
+    Handler for logging to a set of files, which switches from one file
+    to the next when the current file reaches a certain size, or at certain
+    timed intervals
+    """
+    def __init__(self, filename, max_bytes=0, backup_count=0, encoding=None,
+                 delay=0, when='h', interval=1, utc=False, zip_mode=zipfile.ZIP_DEFLATED):
+        handlers.TimedRotatingFileHandler.__init__(self, filename=filename, when=when, interval=interval, utc=utc,
+                                                   backupCount=backup_count, encoding=encoding, delay=delay)
+        self.maxBytes = max_bytes
+        self.zip_mode = zip_mode
+
+    def shouldRollover(self, record):
+        """
+        Determine if rollover should occur.
+        Basically, see if the supplied record would cause the file to exceed
+        the size limit we have.
+        """
+        if self.stream is None:                 # delay was set...
+            self.stream = self._open()
+        if self.maxBytes > 0:                   # are we rolling over?
+            msg = "%s\n" % self.format(record)
+            self.stream.seek(0, 2)              # due to non-posix-compliant Windows feature
+            if self.stream.tell() + len(msg) >= self.maxBytes:
+                return True
+        t = int(time.time())
+        if t >= self.rolloverAt:
+            return True
+        return False
+
+    def find_last_rotated_file(self):
+        dir_name, base_name = os.path.split(self.baseFilename)
+        file_names = os.listdir(dir_name)
+        result = []
+        prefix = f'{base_name}.2'  # we want to find a rotated file with eg filename.2017-12-12... name
+        for file_name in file_names:
+            if file_name.startswith(prefix) and not file_name.endswith('.zip'):
+                result.append(file_name)
+        result.sort()
+        return os.path.join(dir_name, result[0])
+
+    def doRollover(self):
+        super(CompressedSizedTimedRotatingFileHandler, self).doRollover()
+
+        dfn = self.find_last_rotated_file()
+        dfn_zipped = f'{dfn}.zip'
+        if os.path.exists(dfn_zipped):
+            os.remove(dfn_zipped)
+        with zipfile.ZipFile(dfn_zipped, 'w', self.zip_mode) as f:
+            f.write(dfn)
+        os.remove(dfn)
\ No newline at end of file

logging_setup.py

+from settings import LOGGING_CONFIG, DATA_LOGGING_CONFIG
+from os import path, mkdir, statvfs
+from time import gmtime
+import logging
+from compressed_sized_timed_rotating_logger import CompressedSizedTimedRotatingFileHandler
+
+
+def setup_loggers():
+    # Message logging setup
+    log_path = path.join(path.dirname(__file__), 'logs')
+    if not path.isdir(log_path):
+        mkdir(log_path)
+    msg_log_filename = path.join(log_path, 'msg_log')
+    msg_logger = logging.getLogger('msg_logger')
+
+    # Data logging setup
+    base_path = path.dirname(__file__)
+    data_path = path.join(base_path, 'data')
+    if not path.isdir(data_path):
+        mkdir(data_path)
+    data_log_filename = path.join(data_path, 'data_log')
+    data_logger = logging.getLogger('data_logger')
+
+    # Debug and logging
+    debug = LOGGING_CONFIG['debug_mode']
+    if debug:
+        logging_level = logging.DEBUG
+    else:
+        logging_level = logging.INFO
+
+    # Set message logging format and level
+    log_format = '%(asctime)-15s | %(process)d | %(levelname)s: %(message)s'
+    logging_to_console = LOGGING_CONFIG['logging_to_console']
+    msg_handler = CompressedSizedTimedRotatingFileHandler(msg_log_filename, max_bytes=LOGGING_CONFIG['max_bytes'],
+                                                          backup_count=LOGGING_CONFIG['backup_count'],
+                                                          when=LOGGING_CONFIG['when'],
+                                                          interval=LOGGING_CONFIG['interval'])
+    msg_formatter = logging.Formatter(log_format)
+    msg_formatter.converter = gmtime
+    msg_formatter.datefmt = '%Y/%m/%d %H:%M:%S UTC'
+    msg_handler.setFormatter(msg_formatter)
+    msg_logger.addHandler(msg_handler)
+    msg_logger.setLevel(logging_level)
+
+    if logging_to_console:
+        msg_logger.addHandler(logging.StreamHandler())
+
+    # Set data logging level and handler
+    data_logger.setLevel(logging.INFO)
+    data_handler = CompressedSizedTimedRotatingFileHandler(data_log_filename,
+                                                           max_bytes=DATA_LOGGING_CONFIG['max_bytes'],
+                                                           backup_count=DATA_LOGGING_CONFIG['backup_count'],
+                                                           when=DATA_LOGGING_CONFIG['when'],
+                                                           interval=DATA_LOGGING_CONFIG['interval'])
+    data_logger.addHandler(data_handler)
+
+    if not init_logging(msg_logger, logging_level, log_path, data_log_filename):
+        print('ERROR: Could not initialize logging!')
+    return msg_logger, msg_log_filename, data_logger, data_log_filename, logging_level
+
+
+def init_logging(msg_logger, logging_level, log_path, data_log_filename):
+    """ This is the init sequence for the logging system """
+
+    init_logging_status = True
+    msg_logger.info('')
+    msg_logger.info('****************************')
+    msg_logger.info('*** NEW SESSION STARTING ***')
+    msg_logger.info('****************************')
+    msg_logger.info('')
+    msg_logger.info('Logging level: %s' % logging_level)
+    try:
+        st = statvfs('.')
+        available_space = st.f_bavail * st.f_frsize / 1024 / 1024
+        msg_logger.info(f'Remaining disk space : {available_space:.1f} MB')
+    except Exception as e:
+        msg_logger.debug('Unable to get remaining disk space: {e}')
+    msg_logger.info('Saving data log to ' + data_log_filename)
+    msg_logger.info('OhmPi settings:')
+    # TODO Add OhmPi settings
+    msg_logger.info('')
+    msg_logger.info(f'init_logging_status: {init_logging_status}')
+    return init_logging_status

mqtt_setup.py

+from settings import MQTT_CONFIG
+import paho.mqtt.client as mqtt
+
+
+def on_message(client, userdata, message):
+    m = str(message.payload.decode("utf-8"))
+    print(f'message received {m}')
+    print(f'topic: {message.topic}')
+    print(f'qos: {message.qos}')
+    print(f'retain flag: {message.retain}')
+    client.publish(MQTT_CONFIG['measurements_topic'], f'{m} 45 ohm.m')
+
+
+def mqtt_client_setup():
+    client = mqtt.Client(MQTT_CONFIG['client_id'], protocol=5)  # create new client instance
+    client.connect(MQTT_CONFIG['mqtt_broker'])
+    client.on_message = on_message
+    return client, MQTT_CONFIG['measurements_topic']

Ohmpi.py → ohmpi.py

@@ -16,6 +16,8 @@ import time
 from datetime import datetime
 from termcolor import colored
 import threading
+from logging_setup import setup_loggers
+# from mqtt_setup import mqtt_client_setup
 
 # finish import (done only when class is instantiated as some libs are
 # only available on arm64 platform)
@@ -31,11 +33,13 @@ try:
     from digitalio import Direction
     from gpiozero import CPUTemperature
     arm64_imports = True
-except Exception as e:
+except ImportError as e:
     print(f'Warning: {e}')
     arm64_imports = False
 
-
+msg_logger, msg_log_filename, data_logger, data_log_filename, logging_level = setup_loggers()
+# mqtt_client, measurement_topic = mqtt_client_setup()
+# msg_logger.info(f'publishing mqtt to topic {measurement_topic}')
 VERSION = '2.0.1'
 
 print('\033[1m'+'\033[31m'+' ________________________________')
@@ -53,13 +57,6 @@ current_time = datetime.now()
 print(current_time.strftime("%Y-%m-%d %H:%M:%S"))
 
 
-# from logging_setup import setup_loggers
-# from mqtt_setup import mqtt_client_setup
-# msg_logger, msg_log_filename, data_logger, data_log_filename, logging_level = setup_loggers()
-# mqtt_client, measurement_topic = mqtt_client_setup()
-# msg_logger.info(f'publishing mqtt to topic {measurement_topic}')
-
-
 class OhmPi(object):
     """Create the main OhmPi object.
 
@@ -70,8 +67,6 @@ class OhmPi(object):
     sequence : str, optional
         Path to the .txt where the sequence is read. By default, a 1 quadrupole
         sequence: 1, 2, 3, 4 is used.
-    on_pi : bool, optional
-        True if running on the RaspberryPi. False for testing (random data generated).
     output : str, optional
         Either 'print' for a console output or 'mqtt' for publication onto
         MQTT broker.
@@ -87,8 +82,7 @@ class OhmPi(object):
 
         if not arm64_imports:
             self.dump(f'Warning: {e}\n Some libraries only available on arm64 platform could not be imported.\n'
-                  f'The Ohmpi class will fake operations for testing purposes.', 'warning')
-
+                      f'The Ohmpi class will fake operations for testing purposes.', 'warning')
 
         # read in hardware parameters (settings.py)
         self._read_hardware_parameters()
@@ -114,14 +108,6 @@ class OhmPi(object):
         else:
             self.read_quad(sequence)
 
-        # address of the multiplexer board
-        self.board_address = {
-            'A': 0x76,
-            'B': 0x71,
-            'M': 0x74,
-            'N': 0x70
-        }
-
         # connect to components on the OhmPi board
         if self.on_pi:
             # activation of I2C protocol
@@ -198,6 +184,7 @@ class OhmPi(object):
         self.nb_samples = OHMPI_CONFIG['integer']  # number of samples measured for each stack
         self.version = OHMPI_CONFIG['version']  # hardware version
         self.max_elec = OHMPI_CONFIG['max_elec']  # maximum number of electrodes
+        self.board_address = OHMPI_CONFIG['board_address']
         self.dump('OHMPI_CONFIG = ' + str(OHMPI_CONFIG), level='debug')
 
     @staticmethod

requirements.txt

 RPi.GPIO
 adafruit-blinka
 numpy
-pandas
+paho-mqtt
 adafruit-circuitpython-ads1x15
 adafruit-circuitpython-tca9548a
 adafruit-circuitpython-mcp230xx

settings.py

@@ -10,6 +10,7 @@ OHMPI_CONFIG = {
     'integer': 2,  # Max value 10 WHAT IS THIS?
     'version': 2,
     'max_elec': 64,
+    'board_address': {'A': 0x76, 'B': 0x71, 'M': 0x74, 'N': 0x70}  # def. {'A': 0x76, 'B': 0x71, 'M': 0x74, 'N': 0x70}
 }
 
 # local messages logging configuration