rasolar/logger.py

#!/usr/bin/env python

import numpy as np
import os, serial,time,socket,sys,json,logging,requests,getopt
# import the server implementation
# import ADC
#import vedirect
#import upload_osm

configfile="config.json"
cf=open(configfile,"r")
log_conf=json.load(cf)
cf.close()
devicename=socket.gethostname()
if "device" in log_conf:
  devicename=log_conf['device']

mean_count=5
if "mean_count" in log_conf:
  mean_count=int(log_conf['mean_count'])

channel_names=["time","CPU_temp"]
channel_info={"time":{"sensor":"CPU","timestamp":0,"i2c":0},"CPU_temp":{"sensor":"CPU","timestamp":0,"i2c":0}}

a = 2

# import vedirect from https://github.com/karioja/vedirect
# description of channels:https://beta.ivc.no/wiki/index.php/Victron_VE_Direct_DIY_Cable
class vedirect:
    def __init__(self, serialport, timeout):
        self.serialport = serialport
        self.ser = serial.Serial(serialport, 19200, timeout=timeout)
        self.header1 = '\r'
        self.header2 = '\n'
        self.hexmarker = ':'
        self.delimiter = '\t'
        self.key = ''
        self.value = ''
        self.bytes_sum = 0;
        self.state = self.WAIT_HEADER
        self.dict = {}
    (HEX, WAIT_HEADER, IN_KEY, IN_VALUE, IN_CHECKSUM) = range(5)
    def input(self, byte):
        if byte == self.hexmarker and self.state != self.IN_CHECKSUM:
            self.state = self.HEX
        if self.state == self.WAIT_HEADER:
            self.bytes_sum += ord(byte)
            if byte == self.header1:
                self.state = self.WAIT_HEADER
            elif byte == self.header2:
                self.state = self.IN_KEY
            return None
        elif self.state == self.IN_KEY:
            self.bytes_sum += ord(byte)
            if byte == self.delimiter:
                if (self.key == 'Checksum'):
                    self.state = self.IN_CHECKSUM
                else:
                    self.state = self.IN_VALUE
            else:
                self.key += byte
            return None
        elif self.state == self.IN_VALUE:
            self.bytes_sum += ord(byte)
            if byte == self.header1:
                self.state = self.WAIT_HEADER
                self.dict[self.key] = self.value;
                self.key = '';
                self.value = '';
            else:
                self.value += byte
            return None
        elif self.state == self.IN_CHECKSUM:
            self.bytes_sum += ord(byte)
            self.key = ''
            self.value = ''
            self.state = self.WAIT_HEADER
            if (self.bytes_sum % 256 == 0):
                self.bytes_sum = 0
                return self.dict
            else:
                print 'Malformed packet'
                self.bytes_sum = 0
        elif self.state == self.HEX:
            self.bytes_sum = 0
            if byte == self.header2:
                self.state = self.WAIT_HEADER
        else:
            raise AssertionError()
    def read_data(self):
        while True:
            byte = self.ser.read(1)
            packet = self.input(byte)
    def read_data_single(self):
        while True:
            byte = self.ser.read(1)
            packet = self.input(byte)
            if (packet != None):
                return packet
    def read_data_callback(self, callbackFunction):
        while True:
            byte = self.ser.read(1)
            if byte:
                packet = self.input(byte)
                if (packet != None):
                    callbackFunction(packet)
            else:
                break

def print_data_callback(data):
    print data
# end import vedirect

def upload_osm(sensebox_id,sensor_id,value):
	url="https://ingress.opensensemap.org/boxes/%s/%s" % (sensebox_id,sensor_id)
	r = requests.post(url,json={'value': value})
	if (r.status_code != requests.codes.ok) & (r.status_code != 201):
		print("Error %d: %s" % (r.status_code,r.text))

# host and port of internal logging server
bsql=False
if "sqlserver" in log_conf:
  bsql=True
  if "enable" in log_conf['sqlserver']:
    if log_conf['sqlserver']['enable'] == 0:
      bsql=False
  if bsql:
    sqlhost=log_conf['sqlserver']['host']
    sqlport=log_conf['sqlserver']['port']

# config of lux sensor tls2591
btls=False
if "tsl2591" in log_conf:
  btls=True
  if "enable" in log_conf['tsl2591']:
    if log_conf['tsl2591']['enable'] == 0:
      btls=False
  if btls:
    import tsl2591
    tsl_port=1
    if "port" in log_conf['tsl2591']:
      tsl_port=int(log_conf['tsl2591']['port'])
    tsl_add=0x29
    if "address" in log_conf['tsl2591']:
      tsl_add=int(log_conf['tsl2591']['address'],16)
    try:
      tsl = tsl2591.Tsl2591(i2c_bus=tsl_port,sensor_address=tsl_add)  # initialize
    except:
      btls=False
    else:
      channel_names.append("lux")
      channel_info["lux"]={"sensor":"tsl2591","timestamp":0,"i2c":tsl_add}

# config of lux sensor tls2591
bveml=False
if "veml6070" in log_conf:
  bveml=True
  if "enable" in log_conf['veml6070']:
    if log_conf['veml6070']['enable'] == 0:
      bveml=False
  if bveml:
    import veml6070
    veml_port=1
    if "port" in log_conf['veml6070']:
      veml_port=int(log_conf['veml6070']['port'])
    try:
      veml = veml6070.Veml6070(i2c_bus=veml_port)
    except:
      bveml=False
    else:
      channel_names.append("uv")
      channel_info["uv"]={"sensor":"veml6070","timestamp":0,"i2c":0x38}

# config of bme280 sensor
# use pip install RPi.bme280 for library
bbme=False
if "bme280" in log_conf:
  bbme=True
  if "enable" in log_conf['bme280']:
    if log_conf['bme280']['enable'] == 0:
      bbme=False
  if bbme:
    import smbus2
    import bme280
    if "port" in log_conf['bme280']:
      bme_port=log_conf['bme280']['port']
    else:
      bme_port=1
    if "address" in log_conf['bme280']:
      bme_add=int(log_conf['bme280']['address'],16)
    else:
      bme_add=0x77
    try:
      bme_bus=smbus2.SMBus(bme_port)
    except:
      bbme=False
    else:
      calibration_params=bme280.load_calibration_params(bme_bus,bme_add)
      for n in ("temperature","humidity","pressure"):
        channel_names.append(n)
        channel_info[n]={"sensor":"bme280","timestamp":0,"i2c":bme_add}


# configure the client logging
logging.basicConfig()
log = logging.getLogger('./modbus.error')
log.setLevel(logging.ERROR)

# configure tristar
btristar=False
if "tristar" in log_conf:
  btristar=True
  if "enable" in log_conf['tristar']:
    if log_conf['tristar']['enable'] == 0:
      btristar=False
  if btristar:
    import smbus
    from pymodbus.client.sync import ModbusSerialClient as ModbusClient
    tri_port='/dev/ttyUSB0'
    if "port" in log_conf['tristar']:
      tri_port=log_conf['tristar']['port']
    tri_baud=9600
    if "baud" in log_conf['tristar']:
      tri_baud=log_conf['tristar']['baud']
    tri_timeout=1
    if "timeout" in log_conf['tristar']:
      tri_timeout=log_conf['tristar']['timeout']
    try:
      triclient = ModbusClient(method='rtu', port=tri_port, baudrate=tri_baud, timeout=tri_timeout)
    except:
      btristar=False
    else:
      triclient.connect()
      for i in ["volt_scale","amp_scale","volt_bat_term","volt_bat_sens","volt_arr","amp_bat","amp_arr","temp_heatsink","temp_bat","ah_res","ah_tot","kwh_res","kwh_tot","watt_in","watt_out","hour_tot","state","volt_sweep_mp","volt_sweep_oc"]:
        channel_names.append(i)
        channel_info[i]={"sensor":"tristar","timestamp":0,"i2c":0}
      
# declare ve mppt
bve=False
if "vedirect" in log_conf:
  bve=True
  if "enable" in log_conf['vedirect']:
    if log_conf['vedirect']['enable'] == 0:
      bve=False
  if "port" in log_conf['vedirect']:
    bve_port=log_conf['vedirect']['port']
  else:
    bve_port='/dev/serial/by-id/usb-VictronEnergy_BV_VE_Direct_cable_VE1SSBVT-if00-port0'
  print(bve_port)
  try:
    ve=vedirect(bve_port,60)
  except:
    bve=False
  else:
    for i in ["volt_bat_ve","volt_arr_ve","amp_ve","watt_ve","days_ve"]:
      channel_names.append(i)
      channel_info[i]={"sensor":"victron","timestamp":0,"i2c":0}

print(bve)

# declare adc
badc=False
if "ads1x15" in log_conf:
  badc=True
  ads_conf=log_conf['ads1x15']
  if "enable" in ads_conf:
    if ads_conf['enable'] == 0:
      badc=False
  if badc:
    adc={}
    GAIN=1
    if "gain" in ads_conf:
      GAIN=ads_conf['gain']
    if "adc" in ads_conf:
      from Adafruit_ADS1x15 import ADS1115
      from Adafruit_ADS1x15 import ADS1015
      adc_count=0
      tadc=ads_conf['adc']
      for x in tadc:
        y=tadc[x]
        adc_address=0x48
        if "address" in y:
          adc_address=int(y['address'],16)
        abc_bus=1
        if "busnum" in y:
          adc_bus=y['busnum']
        adc_type=1015
        if "type" in y:
          adc_type=y['type']
        adc_assigned=False
        if adc_type == 1115:
          try:
            adc[adc_count]=ADS1115(address=adc_address,busnum=adc_bus)
            adc_assigned=True
          except:
            print("could not assign ADC")
        if adc_type == 1015:
          try:
            adc[adc_count]=ADS1015(address=adc_address,busnum=adc_bus)
            adc_assigned=True
          except:
		    print("could not assign ADC")
        if adc_assigned:
          for j in range(4):
            cnadc="a"+str(adc_count)+"_"+str(j)
            channel_names.append(cnadc)
            channel_info[cnadc]={"sensor":"ads"+str(adc_type),"timestamp":0,"i2c":adc_address}
          adc_count=adc_count+1
    else:
      badc=False

bmcp9808=False
if "mcp9808" in log_conf:
  bmcp9808=True
  if "enable" in log_conf['mcp9808']:
    if log_conf['mcp9808']['enable'] == 0:
      bmcp9808=False
  if bmcp9808:
    import Adafruit_MCP9808.MCP9808 as MCP9808
    try:
      sens_9808 = MCP9808.MCP9808()
    except:
      bmcp9808 = False
    else:
      sens_9808.begin()
      channel_names.append("temp_9808")
      channel_info["temp_9808"]={"sensor":"bmcp9808","timestamp":0,"i2c":0x18}

# push options to internet
bosm=False
if "opensensemap" in log_conf:
  bosm=True
  conf_osm=log_conf['opensensemap']
  if "enable" in conf_osm:
    if conf_osm['enable'] == 0:
      bosm = False
  if bosm:
    try:
      push_count=conf_osm['push_count']
    except:
      push_count=20 # wait 5 cycles till upload to opensensemap
    push_counter=0
    sensebox_id=conf_osm['sensebox_id'] # id of opensensemap
    push_vars=conf_osm['sensors'].keys()
    for pv in conf_osm['sensors'].keys():
      if (pv not in channel_names):
        if len(push_vars)>1:
          push_vars.remove(pv)
        else:
          push_vars=[]
    if (len(push_vars) == 0):
      bosm=False
    else:
      push_data={}
      push_mean_counts={}
      sensebox_sid={}
      for i in push_vars:
        push_data[i]=0
        push_mean_counts[i]=0
        sensebox_sid[i]=conf_osm['sensors'][i]
#luftdaten_id=56009074600018881


ch_val=np.zeros(len(channel_names))
for i in range(len(channel_names)):
	ch_val[i]=(-1)

while a > 1:
  # copy channel values to backup
  ch_old=ch_val.copy()
  
  ch_val=np.zeros(len(channel_names))
  ch_mean=ch_val.copy()
    
  # set actual time
  timestamp=int(1000*time.time())
  ch_val[channel_names.index("time")]=timestamp
  
  for n in range(mean_count):
  # get cpu temperature
    ch_val[channel_names.index("CPU_temp")] = int(open('/sys/class/thermal/thermal_zone0/temp').read())
	  
  # get mcp9808 temperatur
    if bmcp9808:
      ch_val[channel_names.index("temp_9808")] = int(1000*sens_9808.readTempC())
  # get tls lux
    if btls:
      tsl_full,tsl_ir=tsl.get_full_luminosity()
      ch_val[channel_names.index("lux")] = int(1000*tsl.calculate_lux(tsl_full,tsl_ir))
  # get uv index
    if bveml:
      tsl_full,tsl_ir=tsl.get_full_luminosity()
      veml.set_integration_time(veml6070.INTEGRATIONTIME_1T)
#      uv_raw = veml.get_uva_light_intensity_raw()
      ch_val[channel_names.index("uv")] = int(1000*veml.get_uva_light_intensity())

# read adc's
    if badc:
      for i in adc:
        cname="a"+str(i)+"_{0}"
        for j in range(4):
          ch_val[channel_names.index(cname.format(j))]=adc[i].read_adc(j,gain=GAIN)
  
    if btristar:
      try:
        rr = triclient.read_holding_registers(0,90,unit=1)
      except:
        print("could not get data from tristar")
      else:
        try:
          ch_val[channel_names.index("volt_scale")]=rr.registers[0]*65536+rr.registers[1]
        except:
          print("could not read from tristar")
        else:
          ch_val[channel_names.index("amp_scale")]= rr.registers[2]*65536+rr.registers[3]
          ch_val[channel_names.index("volt_bat_term")]=rr.registers[25]
          ch_val[channel_names.index("volt_bat_sens")]=rr.registers[26]
          ch_val[channel_names.index("volt_arr")]=rr.registers[27] # Array voltage
          ch_val[channel_names.index("amp_bat")]=rr.registers[28] # Battery current
          ch_val[channel_names.index("amp_arr")]=rr.registers[29] # Array current
          ch_val[channel_names.index("temp_heatsink")]=rr.registers[35] # Temperature heatsink
          ch_val[channel_names.index("temp_bat")]=rr.registers[36] # Temperature battery
          ch_val[channel_names.index("ah_res")]=rr.registers[52] * 65536 + rr.registers[53] # Ah resetable
          ch_val[channel_names.index("ah_tot")]=rr.registers[54] * 65536 + rr.registers[55] # Ah total
          ch_val[channel_names.index("kwh_res")]=rr.registers[56] # kwh resetable
          ch_val[channel_names.index("kwh_tot")]=rr.registers[57] # kwh total
          ch_val[channel_names.index("watt_in")]=rr.registers[58] # Power in
          ch_val[channel_names.index("watt_out")]=rr.registers[59] # Power out
          ch_val[channel_names.index("hour_tot")]=rr.registers[42] * 65536 + rr.registers[43] # hour total
          ch_val[channel_names.index("state")]=rr.registers[50] # State
          ch_val[channel_names.index("volt_sweep_mp")]=rr.registers[61] # Array voltage
          ch_val[channel_names.index("volt_sweep_oc")]=rr.registers[62] # Array voltage
  
  # read bme280 (temperature, humidity, pressure)
    if bbme:
      bme_data=bme280.sample(bme_bus,bme_add,calibration_params)
      ch_val[channel_names.index("temperature")]=int(1000*bme_data.temperature) # Temperature
      ch_val[channel_names.index("pressure")]=int(1000*bme_data.pressure) # Pressure
      ch_val[channel_names.index("humidity")]=int(1000*bme_data.humidity) # Humidity
  
  # read ve data
    if bve:
      try:
        vedata=ve.read_data_single()
      except:
        print("could not read VE")
      else:
        if 'V' in vedata:
          ch_val[channel_names.index("volt_bat_ve")]=int(vedata['V']) # Battery voltage measured by ve
        if 'VPV' in vedata:
          ch_val[channel_names.index("volt_arr_ve")]=int(vedata['VPV']) # Array voltage measured by ve
        if 'I' in vedata:
          ch_val[channel_names.index("amp_ve")]=int(vedata['I']) # loading current by ve
        if 'PPV' in vedata:
          ch_val[channel_names.index("watt_ve")]=int(vedata['PPV']) # Array power measured by ve
        if 'HSDS' in vedata:
          ch_val[channel_names.index("days_ve")]=int(vedata['HSDS']) # total days online ve
    for i in range(len(ch_val)):
      ch_mean[i]=ch_mean[i]+ch_val[i]
  
  for i in range(len(ch_val)):
    ch_val[i]=int(ch_mean[i]/mean_count)
  
  timefile=round(timestamp/3600000)
  f1=open("/home/pi/log/data_{:d}.txt".format(int(timefile)),"a")
  payload={}
  for i in range(len(ch_val)):
    cni=channel_names[i]
    if ch_val[i] != ch_old[i]:
      f1.write(cni + ":{0};".format(int(ch_val[i])))
      if cni != "time":
        payload[cni]=channel_info[cni]
        payload[cni]['value']=int(ch_val[i])
      channel_info[cni]['timestamp']=timestamp
  f1.write("\n")
  f1.close()
  if bosm:
    for pv in push_vars:
      if push_mean_counts[pv] == push_count:
        sense_data=push_data[pv]/(1000*push_mean_counts[pv])
        if pv == "volt_sweep_oc":
          sense_data=sense_data * ch_val[channel_names.index("volt_scale")] / 65536 / 32768
        upload_osm(sensebox_id,sensebox_sid[pv],round(sense_data,2))
        push_data[pv] = 0
        push_mean_counts[pv] = 0
      else:
        if channel_info[pv]['timestamp'] == timestamp:
          push_data[pv]=push_data[pv]+ch_val[channel_names.index(pv)]
          push_mean_counts[pv]=push_mean_counts[pv]+1
  if bsql:
    json_out={"time":  ch_val[channel_names.index("time")],"device": devicename,"payload":payload}
    try:
      s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    except:
      print("{}: could not connect to database".format(time.time()))
    else:
      try:
        s.connect((sqlhost, sqlport))
      except:
        print("{}: could not connect to database".format(time.time()))
      else:
        s.sendall(json.dumps(json_out))
        s.close()

#  volt_batt=rr.registers[24] * volt_scaling / 65536 / 32768
#  volt_batt_t=rr.registers[25] * volt_scaling / 65536 / 32768
#  volt_batt_sens=rr.registers[26] * volt_scaling / 65536 / 32768
#  volt_arr=rr.registers[27] * volt_scaling / 65536 / 32768
#  curr_batt=rr.registers[28] * amp_scaling / 65536 / 32768
#  curr_arr=rr.registers[29] * amp_scaling / 65536 / 32768
#  temp_heatsink=rr.registers[35]
#  temp_batt=rr.registers[36]
#  ah_reset = rr.registers[52] * 65536 + rr.registers[53] 
#  ah_total = rr.registers[54] * 65536 + rr.registers[55] 
#  kwh_reset = rr.registers[56]
#  kwh_total = rr.registers[57]
#  power_in = rr.registers[58] * volt_scaling * amp_scaling / 131072 / 65536 / 65536 
#  power_out = rr.registers[59] * volt_scaling * amp_scaling / 131072 / 65536 / 65536
#  hourm = rr.registers[42]*65536+rr.registers[43]
#  charge_state = rr.registers[50]
  if bosm:
    if push_counter>=push_count:
      push_counter = 1
    else:
      push_counter=push_counter+1
  
  time.sleep(5)


# close the client
client.close()
f1.close()

print("done")