done
This commit is contained in:
@@ -0,0 +1,6 @@
|
||||
R:\4. Projectgegevens\197_Infraspeed Dyn belasting kunstwerken\09_Producten
|
||||
|
||||
|
||||
use poetry
|
||||
poetry install .
|
||||
poetry run python pdf_less.py #to generate report (it will automatically call the api and based on the time and date of the computer retrieve the correct data and generate the report)
|
||||
Generated
+1359
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,22 @@
|
||||
[tool.poetry]
|
||||
name = "report-builder"
|
||||
version = "0.1.0"
|
||||
description = ""
|
||||
authors = ["Sander <sander@rail1435.nl>"]
|
||||
readme = "README.md"
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "3.12.7"
|
||||
requests = "^2.32.3"
|
||||
tqdm = "^4.66.6"
|
||||
pandas = "^2.2.3"
|
||||
pypdf = "^5.1.0"
|
||||
pikepdf = "^9.4.0"
|
||||
reportlab = "^4.2.5"
|
||||
pillow = "^11.0.0"
|
||||
matplotlib = "^3.9.2"
|
||||
|
||||
|
||||
[build-system]
|
||||
requires = ["poetry-core"]
|
||||
build-backend = "poetry.core.masonry.api"
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,368 @@
|
||||
import requests
|
||||
import time
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
import json
|
||||
from tqdm import tqdm
|
||||
import os
|
||||
from settings import USERNAME,PASSWORD,BASE_URL
|
||||
|
||||
def delay(func,sleep_time=0.1):
|
||||
"""
|
||||
decorator that delays any function
|
||||
|
||||
@partial(delay,sleep_time=1)
|
||||
def func(*args,**kwargs):
|
||||
..
|
||||
return ..
|
||||
"""
|
||||
|
||||
def wrapper(*args,**kwargs):
|
||||
time.sleep(sleep_time)
|
||||
return func(*args,**kwargs)
|
||||
return wrapper
|
||||
|
||||
|
||||
class DataScraper():
|
||||
def __init__(self,username:str,password:str,base_url:str) -> None:
|
||||
self.base_url = base_url
|
||||
self.cookie,self.session_expires = self.login(username,password)["response"]
|
||||
|
||||
def response_formatting(self,response:Any,successfull:bool) -> dict:
|
||||
return {"response":response,"successfull":successfull}
|
||||
|
||||
@partial(delay,sleep_time=0.1)
|
||||
def refresh_session(self) -> None:
|
||||
"""
|
||||
refreshes the expire date of the current token (so no need to update cookie, as the token will be valid for longer)
|
||||
"""
|
||||
requests.post(f"{self.base_url}/refresh",cookies=self.cookie)
|
||||
|
||||
|
||||
@partial(delay,sleep_time=0.1)
|
||||
def login(self,username:str,password:str) -> dict:
|
||||
"""
|
||||
signs in and returns the cookie (with the sesson-token).
|
||||
"""
|
||||
json_obj:dict = {"username":username,"password":password}
|
||||
|
||||
response = requests.post(f"{self.base_url}/login",json=json_obj)
|
||||
|
||||
if(not response.ok):
|
||||
print(f"response:{response}")
|
||||
print(f"response.content:{response.content}")
|
||||
print(f"response.cookies:{response.cookies}")
|
||||
exit("couldn't successfully login")
|
||||
|
||||
session_expires = json.loads(response.content.decode("utf-8"))["expire"]
|
||||
return self.response_formatting((response.cookies,session_expires),True)
|
||||
|
||||
@partial(delay,sleep_time=0.1)
|
||||
def get_all_sensor_names(self) -> tuple[list[str],bool]:
|
||||
response = requests.get(f"{self.base_url}/api/sensor/getallnames",cookies=self.cookie)
|
||||
|
||||
if(response.ok):
|
||||
#only return the names
|
||||
return self.response_formatting([elem["name"] for elem in json.loads(response.content.decode("utf-8"))["names"]],True)
|
||||
else:
|
||||
return self.response_formatting(response.content,False)
|
||||
|
||||
|
||||
@partial(delay,sleep_time=0.1)
|
||||
def get_data(self,name:str,history_length:int,bin_size:int=1) -> dict:
|
||||
"""
|
||||
History length in days
|
||||
Bin size is used to reduce the amount of data that is send, it would bin the data in bins of n minutes and only send over the average within the bin.
|
||||
"""
|
||||
|
||||
json_obj = {"name":name,"binsize":bin_size,"length":history_length}
|
||||
|
||||
response = requests.get(f"{self.base_url}/api/sensor/getdata",params=json_obj,cookies=self.cookie)
|
||||
|
||||
if(response.ok):
|
||||
return self.response_formatting(response.content, True)
|
||||
else:
|
||||
return self.response_formatting(response.content, False)
|
||||
|
||||
@partial(delay,sleep_time=0.1)
|
||||
def get_info_sensor(self,name:str) -> dict:
|
||||
"""
|
||||
returns information related to the sensor
|
||||
"createdAt", moment the sensor is created in db
|
||||
"fw_version", #firmware version
|
||||
"imei", #unique indentifier
|
||||
"is_active", #if the sensor is active or not
|
||||
"latitude", #where it's placed geographically
|
||||
"longitude",
|
||||
"name", #unique name
|
||||
"notes",
|
||||
"project_id", #id of the project it belongs to
|
||||
"status", #status code
|
||||
"type", #type sensor BS1000 of BS1400, etc..
|
||||
"updatedAt" #last update of content in the database
|
||||
"""
|
||||
|
||||
json_obj = {"name":name}
|
||||
|
||||
response = requests.get(f"{self.base_url}/api/sensor/get",params=json_obj,cookies=self.cookie)
|
||||
|
||||
if(response.ok):
|
||||
return self.response_formatting(response.content,True)
|
||||
else:
|
||||
return self.response_formatting(response.content,False)
|
||||
|
||||
@partial(delay,sleep_time=0.1)
|
||||
def get_all_sensors(self):
|
||||
"""
|
||||
Retrieves a list of all sensors in the database. (the information retrieved is information from the sensor)
|
||||
"""
|
||||
|
||||
response = requests.get(f"{self.base_url}/api/sensor/getall",cookies=self.cookie)
|
||||
|
||||
if(response.ok):
|
||||
return self.response_formatting(response.content,True)
|
||||
else:
|
||||
return self.response_formatting(response.content,False)
|
||||
|
||||
@partial(delay,sleep_time=0.1)
|
||||
def get_predictions(self,name:str):
|
||||
"""
|
||||
name: sensor name
|
||||
|
||||
Fetches predictions from the database based on optional query parameters for sensor name, time start, and time end. If parameters are not provided, it retrieves all predictions.
|
||||
"""
|
||||
json_obj = {"sensorname":name}
|
||||
|
||||
response = requests.get(f"{self.base_url}/api/learning/prediction/get",params=json_obj,cookies=self.cookie)
|
||||
|
||||
if(response.ok):
|
||||
return self.response_formatting(response.content,True)
|
||||
else:
|
||||
return self.response_formatting(response.content,False)
|
||||
|
||||
@partial(delay,sleep_time=0.1)
|
||||
def get_labels(self,name:str):
|
||||
"""
|
||||
name: sensorname
|
||||
|
||||
Retrieves classification labels for a specific sensor based on the sensorname query parameter.
|
||||
|
||||
"""
|
||||
json_obj = {"sensorname":name}
|
||||
response = requests.get(f"{self.base_url}/api/classification/labels/get",params=json_obj,cookies=self.cookie)
|
||||
|
||||
if(response.ok):
|
||||
return self.response_formatting(response.content,True)
|
||||
else:
|
||||
return self.response_formatting(response.content,False)
|
||||
|
||||
@partial(delay,sleep_time=0.1)
|
||||
def get_all_sensor_IMEIs(self) -> list[str]:
|
||||
response = requests.get(f"{self.base_url}/api/sensor/getall",cookies=self.cookie)
|
||||
|
||||
if(response.ok):
|
||||
|
||||
sensor_dict = json.loads(response.content.decode("utf-8"))
|
||||
|
||||
|
||||
EMEIs = [sensor_dict[key]["imei"] for key in sensor_dict.keys()]
|
||||
|
||||
return self.response_formatting(EMEIs,True)
|
||||
else:
|
||||
return self.response_formatting(response.content,False)
|
||||
|
||||
@partial(delay,sleep_time=0.1)
|
||||
def getLabeled_data(self,sensor_name:str,history_length:int=365,bin_size:int=1):
|
||||
json_obj = {"name":sensor_name,"binsize":bin_size,"length":history_length}
|
||||
|
||||
response = requests.get(f"{self.base_url}/api/sensor/getlabeleddata",cookies=self.cookie,params=json_obj)
|
||||
if(response.ok):
|
||||
return self.response_formatting(response.content,True)
|
||||
else:
|
||||
return self.response_formatting(response.content,False)
|
||||
|
||||
class Info_storage():
|
||||
def __init__(self,scraper:DataScraper,data_folder_name:str = "data") -> None:
|
||||
self.scraper = scraper
|
||||
self.sensor_IMEIs = [] #list of imeis
|
||||
self.sensor_info = [] #list of {imei:name,sensor_info:sensor_info}
|
||||
self.sensor_names = [] #list of names
|
||||
self.sensor_data = [] #list of {imei:sensor_data}
|
||||
self.sensor_predictions = [] #list of {name:sensor_predictions}
|
||||
self.sensor_classification_labels = [] #list of {name:sensor_prediction_labels}
|
||||
self.sensor_data_labeled = [] #list of {sensor_name:sensor_data}
|
||||
|
||||
self.data_folder_name = data_folder_name
|
||||
self.data_folder_path = os.path.join(os.curdir,self.data_folder_name)
|
||||
|
||||
|
||||
def write_to_file(self,file_name:str,jsonifiable_data:Any) -> None:
|
||||
"""
|
||||
writes jsonifiable data to a file
|
||||
"""
|
||||
with open(os.path.join(self.data_folder_path,file_name),"wt") as f:
|
||||
try:
|
||||
obj = json.dumps(jsonifiable_data) #str(json.loads(str(jsonifiable_data).strip("'<>() ").replace('\'', '\"').replace('\0', '').replace("True","true").replace("False","false")))
|
||||
f.write(obj)
|
||||
|
||||
except Exception as e:
|
||||
print(e)
|
||||
print(f"CANT SAVE: {self.sensor_names}")
|
||||
|
||||
|
||||
def read_from_file(self,file_name:str) -> Any:
|
||||
sensor_names_path = os.path.join(self.data_folder_path,file_name)
|
||||
with open(sensor_names_path) as f:
|
||||
return json.loads(f.readline().replace("'",'"'))
|
||||
|
||||
def retrieve(self,use_imei:bool=True) -> None:
|
||||
"""
|
||||
new retrieve data from server
|
||||
"""
|
||||
#get sensor names
|
||||
self.sensor_names = self.scraper.get_all_sensor_names()["response"]
|
||||
self.sensor_IMEIs = self.scraper.get_all_sensor_IMEIs()["response"]
|
||||
|
||||
#get sensor info
|
||||
sensor_list = self.sensor_names
|
||||
if(use_imei):
|
||||
sensor_list = self.sensor_IMEIs
|
||||
|
||||
|
||||
for index,name in enumerate(tqdm(sensor_list,ascii=True,desc="get_sensor_info")):
|
||||
resp = self.scraper.get_info_sensor(name)
|
||||
if(resp["successfull"]):
|
||||
sensor_info = json.loads(resp["response"].decode("utf-8"))
|
||||
self.sensor_info.append({"name":name,"sensor_info":sensor_info})
|
||||
else:
|
||||
print(f"name: {name}, unsuccessfull: {resp}")
|
||||
|
||||
if(index % 100 == 0): #refresh session if it takes long
|
||||
self.scraper.refresh_session()
|
||||
|
||||
#sensor data
|
||||
for index,name in enumerate(tqdm(sensor_list,ascii=True,desc="sensor_data")):
|
||||
resp = self.scraper.get_data(name,history_length=364,bin_size=1)
|
||||
|
||||
if(resp["successfull"]):
|
||||
sensor_data = json.loads(resp["response"].decode("utf-8"))
|
||||
self.sensor_data.append({"name":name,"sensor_data":sensor_data})
|
||||
else:
|
||||
print(f"unsuccessfull: {resp}")
|
||||
|
||||
if(index % 100 == 0):
|
||||
self.scraper.refresh_session()
|
||||
|
||||
#predictions
|
||||
for index,name in enumerate(tqdm(self.sensor_names,ascii=True,desc="predictions")):
|
||||
resp = self.scraper.get_predictions(name)
|
||||
if(resp["successfull"]):
|
||||
sensor_pred = json.loads(resp["response"].decode("utf-8"))
|
||||
self.sensor_predictions.append({"name":name,"sensor_pred":sensor_pred})
|
||||
else:
|
||||
print(f"unsuccessfull: {resp}")
|
||||
|
||||
if(index % 100 == 0):
|
||||
self.scraper.refresh_session()
|
||||
|
||||
|
||||
#classification labels
|
||||
for index,name in enumerate(tqdm(self.sensor_names,ascii=True,desc="classification_labels")):
|
||||
resp = self.scraper.get_predictions(name)
|
||||
if(resp["successfull"]):
|
||||
sensor_classification_labels = json.loads(resp["response"].decode("utf-8"))
|
||||
self.sensor_classification_labels.append({"name":name,"sensor_classification_labels":sensor_classification_labels})
|
||||
else:
|
||||
print(f"unsuccessfull: {resp}")
|
||||
|
||||
if(index % 100 == 0):
|
||||
self.scraper.refresh_session()
|
||||
|
||||
for index,name in enumerate(tqdm(self.sensor_names,ascii=True,desc="labeled data")):
|
||||
resp = self.scraper.getLabeled_data(name)
|
||||
if(resp["successfull"]):
|
||||
labeled_data = json.loads(resp["response"].decode("utf-8"))
|
||||
self.sensor_data_labeled.append({"name":name,"labeled_data":labeled_data})
|
||||
else:
|
||||
print(f"unsuccessfull: {resp}")
|
||||
|
||||
if(index % 100 == 0):
|
||||
self.scraper.refresh_session()
|
||||
|
||||
def save(self) -> None:
|
||||
"""
|
||||
saves all of the data to files
|
||||
"""
|
||||
if(not os.path.isdir(self.data_folder_name)):
|
||||
os.makedirs(self.data_folder_path)
|
||||
|
||||
#store sensor names
|
||||
self.write_to_file("sensor_names.txt",self.sensor_names)
|
||||
|
||||
#store sensor EMEI's
|
||||
self.write_to_file("sensor_EMEIs.txt",self.sensor_IMEIs)
|
||||
|
||||
#store sensor info
|
||||
self.write_to_file("sensor_info.txt",self.sensor_info)
|
||||
|
||||
#store sensor data
|
||||
self.write_to_file("sensor_data.txt",self.sensor_data)
|
||||
|
||||
#store sensor predictions
|
||||
self.write_to_file("sensor_predictions.txt",self.sensor_predictions)
|
||||
|
||||
#store sensor classification labels
|
||||
self.write_to_file("sensor_classification_labels.txt",self.sensor_classification_labels)
|
||||
|
||||
#store labeled data
|
||||
self.write_to_file("labeled_data.txt",self.sensor_data_labeled)
|
||||
|
||||
|
||||
def load(self) -> None:
|
||||
"""
|
||||
load data from local
|
||||
"""
|
||||
#load sensor names
|
||||
self.sensor_names = self.read_from_file("sensor_names.txt")
|
||||
|
||||
#store sensor EMEI's
|
||||
self.sensor_IMEIs = self.read_from_file("sensor_EMEIs.txt")
|
||||
|
||||
#load sensor info
|
||||
self.sensor_info = self.read_from_file("sensor_info.txt")
|
||||
|
||||
#load sensor data
|
||||
self.sensor_data = self.read_from_file("sensor_data.txt")
|
||||
|
||||
#load sensor predictions
|
||||
self.sensor_predictions = self.read_from_file("sensor_predictions.txt")
|
||||
|
||||
#load sensor classification labels
|
||||
self.sensor_classification_labels = self.read_from_file("sensor_classification_labels.txt")
|
||||
|
||||
#load labeled data
|
||||
self.sensor_data_labeled = self.read_from_file("labeled_data.txt")
|
||||
|
||||
|
||||
"""
|
||||
Since we got duplicates with the names, we will have to try to implement it using the IMEIs,
|
||||
TODO currently no sensor data is coming through when using IMEIS, figure out why this is the case.
|
||||
|
||||
"""
|
||||
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
USE_IMEIS = False #NOTE if you get by IMEIS it will not do the conversion and you'll only get the data in SV1,SV2,SV3,SV4,SV5,SV6 as it depends on the sensor what kind of data will be on SV_n. (the type of the sensor can be derived from the name, and is done so to do the conversion)
|
||||
|
||||
|
||||
scraper = DataScraper(username=USERNAME,password=PASSWORD,base_url=BASE_URL)
|
||||
|
||||
storage = Info_storage(scraper)
|
||||
|
||||
storage.retrieve(use_imei=USE_IMEIS)
|
||||
|
||||
storage.save()
|
||||
storage.load()
|
||||
|
||||
@@ -0,0 +1,113 @@
|
||||
import PIL
|
||||
from PIL import Image ,ImageDraw, ImageFont
|
||||
import matplotlib.pyplot as plt
|
||||
from getData import DataScraper
|
||||
import pandas as pd
|
||||
from settings import USERNAME,PASSWORD,BASE_URL
|
||||
from datetime import date
|
||||
import json
|
||||
from tqdm import tqdm
|
||||
from collections import OrderedDict
|
||||
import numpy as np
|
||||
from copy import deepcopy
|
||||
from datetime import datetime
|
||||
|
||||
def get_data(emeis:list[str],scraper:DataScraper) -> list[pd.DataFrame]:
|
||||
dataframes = []
|
||||
for imei in tqdm(emeis,ascii=True,desc="gather_data"):
|
||||
resp = json.loads(scraper.get_data(imei,history_length=90)["response"].decode("utf-8"))["data"]
|
||||
df = pd.DataFrame(resp)
|
||||
df['MeasurementTime'] = pd.to_datetime(df['MeasurementTime']) #fix time date
|
||||
dataframes.append(df)
|
||||
|
||||
return dataframes
|
||||
|
||||
|
||||
def get_data_within_n_days(df:pd.DataFrame,num_days:int,current_date:str) -> pd.DataFrame:
|
||||
"""
|
||||
takes the data from n days back. where 0 means only the current day. 1 means current and yesterday, etc.
|
||||
|
||||
"""
|
||||
cutoff_date = pd.Timestamp(current_date).tz_localize('UTC') -pd.Timedelta(days=num_days) #'2023-10-01'
|
||||
|
||||
filtered_df = df[df['MeasurementTime'] >= cutoff_date]
|
||||
|
||||
return filtered_df
|
||||
|
||||
def get_data_after_date(df:pd.DataFrame,after_date:datetime=datetime(2024,10,28)) -> pd.DataFrame:
|
||||
|
||||
return df[df['MeasurementTime'] >= pd.Timestamp(str(after_date)).tz_localize('UTC')]
|
||||
|
||||
|
||||
def add_elem(text:str,x:int,y:int,img,font_size:int=28):
|
||||
draw = ImageDraw.Draw(img)
|
||||
font = ImageFont.truetype("arial.ttf", font_size)
|
||||
draw.text((x, y), text, font=font, fill=(int(0.37254901960784315*255),int(0.20784313725490197*255),int(0.09411764705882353*255)))
|
||||
return img
|
||||
|
||||
|
||||
def fill_column(elements:list[str],column_index:int,img):
|
||||
"""
|
||||
column index starts at 0 for the first column
|
||||
"""
|
||||
column_shift = [900,1100,1330,1900]
|
||||
dist = 37
|
||||
row_shift = [720,760,925,1120,1295,1470,1510,1650,1650+dist,1650+dist*2,1650+dist*3,1650+dist*4,1650+dist*5,1650+dist*6,1650+dist*7,1650+dist*8,1650+dist*9,2135,2315,2315+dist*1,2315+dist*2,2315+dist*3,2560,2750,2750+dist]
|
||||
|
||||
for row_index,elem in enumerate(elements):
|
||||
add_elem(elem,column_shift[column_index],row_shift[row_index],img)
|
||||
|
||||
return img
|
||||
|
||||
def df_slope(df:pd.DataFrame,y_column_name:str) -> tuple:
|
||||
"""
|
||||
assumes that 'MeasurementTime' carries the DateTime data.
|
||||
"""
|
||||
|
||||
df["x"] = (df["MeasurementTime"]-df["MeasurementTime"].min()).dt.total_seconds()/(60*60*4) #resolution of an hour
|
||||
slope, intercept = np.polyfit(df["x"] , df[y_column_name], 1)
|
||||
return slope, intercept
|
||||
|
||||
"""
|
||||
Sv4 dynamisch
|
||||
Sv5 static
|
||||
"""
|
||||
if __name__ == "__main__":
|
||||
pd.options.mode.copy_on_write = True
|
||||
current_date = date.today()
|
||||
week_number = date.isocalendar(current_date)[1]
|
||||
|
||||
scraper = DataScraper(USERNAME,PASSWORD,BASE_URL)
|
||||
imeis = OrderedDict([("KW1100-0070","86620705863754"),("KW1100-0071","86620705863410"),("KW1100-0080", "86620705863765"),("KW1100-0069","86620705863772"),("KW1100-0065","86620705864160"),("KW1100-0078","86620705863735"),("KW1100-0061","86620705863782"),("KW1100-0059","86620705863392"),("KW1100-0062","86620705863715"),("KW1100-0075","86620705863575"),("KW1100-0073","86620705864180"),("KW1100-0064","86620705863719"),("KW1100-0063","86620705863729"),("KW1100-0066","86620705863736"),("KW1100-0068","86620705864119"),("KW1100-0054","86366306494401"),("KW1100-0074","86620705864162"),("KW1100-0060","86620705863639"),("KW1100-0055","86620705863526"),("KW1100-0058","86620705864178"),("KW1100-0057","86620705864123") , ("KW1100-0056","86620705864099"),("KW1100-0051","86366306494409"),("KW1100-0079","86620705864105"),("KW1100-0049","86366306491069")])
|
||||
img = Image.open("template.png")
|
||||
img = add_elem(str(current_date),400,160,img,font_size=40)
|
||||
img = add_elem(str(week_number),400,110,img,font_size=40)
|
||||
|
||||
|
||||
|
||||
dfs = get_data(list(imeis.values()),scraper)
|
||||
|
||||
dfs = [get_data_after_date(df) for df in dfs] #cull the herd of old data
|
||||
|
||||
seven_day_median = [str(round(get_data_within_n_days(df,7,current_date)["Sv4"].median(),1)) for df in dfs]
|
||||
ninety_day_median = [str(round(get_data_within_n_days(df,90,current_date)["Sv4"].median(),1)) for df in dfs]
|
||||
|
||||
img = fill_column(seven_day_median,0,img) #7 + current day
|
||||
img = fill_column(ninety_day_median,1,img) #90 + current day
|
||||
|
||||
#week trend
|
||||
slope = [str(round(df_slope(get_data_within_n_days(df,7,current_date),"Sv4")[0],1)) for df in dfs]
|
||||
img = fill_column(slope,2,img)
|
||||
|
||||
#static delta 7 daags
|
||||
deltas = []
|
||||
for df in dfs:
|
||||
x = get_data_within_n_days(df,7,current_date)["Sv4"]
|
||||
deltas.append(str(round(x.max()-x.min(),1)))
|
||||
|
||||
|
||||
img = fill_column(deltas,3,img)
|
||||
# plt.imshow(img)
|
||||
# plt.show()
|
||||
img.save("report.pdf")
|
||||
|
||||
+198461
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,3 @@
|
||||
USERNAME = "admin"
|
||||
PASSWORD = "WyBNFj3rhNWtxpR"
|
||||
BASE_URL = "https://data.rail1435.nl/"
|
||||
Binary file not shown.
Binary file not shown.
|
After Width: | Height: | Size: 515 KiB |
Reference in New Issue
Block a user