Fajny gustaw

backend/kst_app/__init__.py

@@ -13,3 +13,4 @@
 
 with app.app_context():
     db.create_all()
+

backend/kst_app/data_processing/__init__.py

@@ -0,0 +1,2 @@
+from .data_miner import mine_ids_sorted
+from .batcher import batch_sensor_data_day, batch_auxiliary_data_day, batch_sensor_data_week, batch_auxiliary_data_week

backend/kst_app/data_processing/batcher.py

@@ -0,0 +1,90 @@
+from typing import List
+import numpy as np
+from kst_app.data_storage.models import Measurement, WeatherMeasurement
+
+
+def batch_sensor_data_day(measurment_list: List[Measurement]) -> np.array:
+    assert len(measurment_list) >= 24 * 50, 'not enough data'  # The number of measurments in one day
+    last_24 = measurment_list[:24 * 50]
+
+    batched_sensors = np.zeros([1, 24, 150])
+    counter_sensors = 0
+    counter_hours = 0
+    for measurment in last_24:
+        batched_sensors[0, counter_hours, counter_sensors * 3:(counter_sensors + 1) * 3] = [measurment.pm1,
+                                                                                            measurment.pm25,
+                                                                                            measurment.pm10]
+        counter_sensors += 1
+        if counter_sensors == 50:
+            counter_sensors = 0
+            counter_hours += 1
+
+    return batched_sensors
+
+def batch_sensor_data_week(measurment_list: List[Measurement], idx:int = None):
+    if idx is None:
+        last_week = measurment_list[-24*50*7:]
+    else:
+        last_week = measurment_list[idx:idx+24*50*7]
+    assert len(last_week) >= 24 * 50 * 7, 'not enough data'
+
+    batched_sensors = np.zeros([1, 24 * 7, 150])
+    counter_sensors = 0
+    counter_hours = 0
+    for measurment in last_week:
+        batched_sensors[0, counter_hours, counter_sensors * 3:(counter_sensors + 1) * 3] = [measurment.pm1,
+                                                                                            measurment.pm25,
+                                                                                            measurment.pm10]
+        counter_sensors += 1
+        if counter_sensors == 50:
+            counter_sensors = 0
+            counter_hours += 1
+
+    return batched_sensors
+
+
+
+
+def batch_auxiliary_data_day(measurment_list: List[WeatherMeasurement]) -> np.array:
+    assert len(measurment_list) >= 24, 'not enough weather data'
+    last_24 = measurment_list[0:24]
+
+    batched_weather = np.zeros((1, 24, 10))
+
+    for i, measurment in enumerate(last_24):
+        batched_weather[0, i] = [measurment.rainLevel,
+                                 measurment.temperature,
+                                 measurment.humidity,
+                                 measurment.pressure,
+                                 measurment.windDirection,
+                                 measurment.windSpeed,
+                                 measurment.date.hour,
+                                 measurment.date.day,
+                                 measurment.date.month,
+                                 measurment.date.year]
+
+    return batched_weather
+
+def batch_auxiliary_data_week(measurment_list: List[WeatherMeasurement], idx:int = None) -> np.array:
+    if idx is None:
+        last_week = measurment_list[-24*7:]
+    else:
+        last_week = measurment_list[idx:idx+24*7]
+
+    assert len(last_week) >= 24 * 7, 'not enough weather data'
+
+    batched_weather = np.zeros((1, 24*7, 10))
+
+    for i, measurment in enumerate(last_week):
+        batched_weather[0, i] = [measurment.rainLevel,
+                                 measurment.temperature,
+                                 measurment.humidity,
+                                 measurment.pressure,
+                                 measurment.windDirection,
+                                 measurment.windSpeed,
+                                 measurment.date.hour,
+                                 measurment.date.day,
+                                 measurment.date.month,
+                                 measurment.date.year]
+
+    return batched_weather

backend/kst_app/data_processing/data_miner.py

@@ -0,0 +1,8 @@
+from typing import List
+from kst_app.data_storage.models import get_all_sensors
+
+
+def mine_ids_sorted() -> List[int]:
+    sensors = get_all_sensors()
+    sensors = sorted(sensors, key=lambda sensor: sensor.id)
+    return [sensor.id for sensor in sensors]

backend/kst_app/data_storage/config.py

@@ -0,0 +1,2 @@
+DEBUG = True  # Turns on debugging features in Flask
+SQLALCHEMY_DATABASE_URI = "mysql+mysqldb://root:secret_password@localhost:3306/kst_db"

backend/kst_app/data_storage/models.py

@@ -11,6 +11,7 @@
 
 
 class Sensor(db.Model):
+    __table_args__ = {'extend_existing': True}
     id = db.Column(db.Integer, primary_key=True)
     longitude = db.Column(db.Float(precision=10))
     latitude = db.Column(db.Float(precision=10))
@@ -81,6 +82,7 @@ def update_sensor(sensor_id: int, longitude: float, latitude: float):
 
 
 class Measurement(db.Model):
+    __table_args__ = {'extend_existing': True}
     id = db.Column(db.Integer, primary_key=True)
     pm1 = db.Column(db.Float(precision=10))
     pm25 = db.Column(db.Float(precision=10))
@@ -135,7 +137,13 @@ def get_measurement(measurement_id: int) -> Measurement:
 
 
 def get_all_measurements() -> List[Measurement]:
-    return Measurement.query.all()
+    return Measurement.query.order_by(Measurement.date, Measurement.sensor_id).all()
+
+
+def delete_all_measurements() -> int:
+    deleted = Measurement.query.delete()
+    db.session.commit()
+    return deleted
 
 
 def delete_measurement(measurement_id: int) -> Measurement:
@@ -176,6 +184,7 @@ def update_measurement(measurement_id: int, pm1: float, pm25: float, pm10: float
 
 
 class PredictedMeasurement(db.Model):
+    __table_args__ = {'extend_existing': True}
     id = db.Column(db.Integer, primary_key=True)
     pm1 = db.Column(db.Float(precision=10))
     pm25 = db.Column(db.Float(precision=10))
@@ -221,7 +230,7 @@ def get_predicted_measurement(measurement_id: int) -> PredictedMeasurement:
 
 
 def get_all_predicted_measurements() -> List[PredictedMeasurement]:
-    return PredictedMeasurement.query.all()
+    return PredictedMeasurement.query.order_by(PredictedMeasurement.date, PredictedMeasurement.sensor_id).all()
 
 
 def delete_predicted_measurement(measurement_id: int) -> PredictedMeasurement:
@@ -231,6 +240,12 @@ def delete_predicted_measurement(measurement_id: int) -> PredictedMeasurement:
     return predicted_measurement
 
 
+def delete_all_predicted_measurements() -> List[PredictedMeasurement]:
+    deleted = PredictedMeasurement.query.delete()
+    db.session.commit()
+    return deleted
+
+
 def update_predicted_measurement(measurement_id: int, pm1: float, pm25: float, pm10: float,
                                  date: datetime, sensor_id: int) -> PredictedMeasurement:
     measurement = PredictedMeasurement.query.filter_by(id=measurement_id).one()
@@ -254,6 +269,7 @@ def update_predicted_measurement(measurement_id: int, pm1: float, pm25: float, p
 
 
 class WeatherMeasurement(db.Model):
+    __table_args__ = {'extend_existing': True}
     id = db.Column(db.Integer, primary_key=True)
     temperature = db.Column(db.Float(precision=10))
     humidity = db.Column(db.Float(precision=10))
@@ -310,7 +326,9 @@ def get_weather_measurement(measurement_id: int) -> WeatherMeasurement:
 
 
 def get_all_weather_measurements() -> List[WeatherMeasurement]:
-    return WeatherMeasurement.query.all()
+    deleted =  WeatherMeasurement.query.order_by(WeatherMeasurement.date).all()
+    db.session.commit()
+    return deleted
 
 
 def delete_weather_measurement(measurement_id: int) -> WeatherMeasurement:
@@ -320,6 +338,10 @@ def delete_weather_measurement(measurement_id: int) -> WeatherMeasurement:
     return weather_measurement
 
 
+def delete_all_weather_measurement() -> int:
+    return WeatherMeasurement.query.delete()
+
+
 def update_weather_measurement(measurement_id: int, temperature: float, humidity: float, pressure: float,
                                windSpeed: float, windDirection: float,
                                rainLevel: float, date: datetime, sensor_id: int) -> WeatherMeasurement:

backend/kst_app/data_storage/models_test.py

@@ -0,0 +1,16 @@
+import unittest
+import pickle
+from .models import Measurement, add_measurements
+
+class TestModels(unittest.TestCase):
+    def add_measurements_test(self):
+        with open('C:\\Users\\bartw\\OneDrive\\Pulpit\\KST\\backend\\kst_app\\data.pkl', 'rb') as port:
+            data = pickle.load(port)
+
+        list_of_measurments = []
+        for key, value in data.items():
+            for id, info in value.items():
+                measurment = Measurement(info[0], info[1], info[2], 0, 0, 0, key, id)
+                list_of_measurments.append(measurment)
+
+        add_measurements(list_of_measurments)

backend/kst_app/engine/TFModel.py

@@ -0,0 +1,25 @@
+import tensorflow as tf
+
+model = tf.keras.Model
+
+
+def load_model():
+    global model
+    model = tf.keras.models.load_model('model1h.h5')
+
+
+def save_model():
+    model.save('model1h.h5')
+    tf.keras.backend.clear_session()
+
+
+def load_gustaw():
+    def alst_time_step_mse(Y_true, Y_pred):
+        return tf.keras.metrics.mean_squared_error(Y_true[:, -1], Y_pred[:, -1])
+
+    return tf.keras.models.load_model('gustaw.h5',custom_objects={"alst_time_step_mse": alst_time_step_mse})
+
+
+def save_gustaw(gustaw: tf.keras.Model):
+    gustaw.save('gustaw.h5')
+    tf.keras.backend.clear_session()

backend/kst_app/engine/__init__.py

@@ -0,0 +1 @@
+from .predictor import predict_hour

backend/kst_app/engine/model_architecture.py

@@ -0,0 +1,22 @@
+import tensorflow as tf
+
+def create_clean_model_hour():
+    input_rec = tf.keras.layers.Input((24, 150))
+    norm = tf.keras.layers.Normalization()(input_rec)
+    LSTM1 = tf.keras.layers.LSTM(800, return_sequences=True)(norm)
+    LSTM2 = tf.keras.layers.LSTM(800, dropout=.1)(LSTM1)
+
+    inputs_aux = tf.keras.layers.Input((24, 10))
+    flatten = tf.keras.layers.Flatten()(inputs_aux)
+    pre_preprocess = tf.keras.layers.Dense(256, activation='elu', kernel_regularizer='l2')(flatten)
+    pre_preprocess = tf.keras.layers.Dense(256, activation='elu', kernel_regularizer='l2')(pre_preprocess)
+    preprocess = tf.keras.layers.Dense(128, activation='elu', kernel_regularizer='l2')(pre_preprocess)
+
+    combine = tf.keras.layers.Concatenate()([preprocess, LSTM2])
+    broad_layer = tf.keras.layers.Dense(300, activation='elu', kernel_regularizer='l2')(combine)
+    out = tf.keras.layers.Dense(150, activation='linear')(broad_layer)
+
+    model = tf.keras.Model([input_rec, inputs_aux], out)
+    model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001), metrics=['mse'])
+
+    return model

backend/kst_app/engine/predictor.py

@@ -0,0 +1,88 @@
+from kst_app.engine.TFModel import model, load_model, save_model, load_gustaw, save_gustaw
+import kst_app.data_processing as process
+from kst_app.data_storage import models as m
+from kst_app import app
+from typing import List, Tuple
+import numpy as np
+from datetime import datetime, timedelta
+
+
+def predict_hour():
+    data, weather = fetch_data()
+    batched_data, batched_weather = batch_data_day(data, weather)
+
+    last_measurment_date = weather[0].date
+
+    load_model()
+    out = model.predict([batched_data, batched_weather])
+    save_model()
+
+    save_record(out, last_measurment_date)
+
+
+def predict_12h():
+    delete_previous_predictions()
+
+    data, weather = fetch_data()
+    batched_data, batched_weather = batch_data_week(data, weather)
+
+    last_measurment_date = weather[-1].date
+
+    model = load_gustaw()
+    out = model.predict([batched_data, batched_weather])
+    save_gustaw(model)
+
+    save_record_week(out, last_measurment_date)
+
+
+def fetch_data() -> Tuple[List[m.Measurement], List[m.WeatherMeasurement]]:
+    return m.get_all_measurements(), m.get_all_weather_measurements()
+
+def delete_previous_predictions():
+    m.delete_all_predicted_measurements()
+
+
+def batch_data_day(data: List[m.Measurement], weather: List[m.WeatherMeasurement]) -> Tuple[np.array, np.array]:
+    return process.batch_sensor_data_day(data), process.batch_auxiliary_data_day(weather)
+
+
+def batch_data_week(data: List[m.Measurement], weather: List[m.WeatherMeasurement],idx:int = None) -> Tuple[np.array, np.array]:
+    return process.batch_sensor_data_week(data,idx), process.batch_auxiliary_data_week(weather,idx)
+
+
+def save_record(record: List[float], date: datetime):
+    predicted_date = date.hour + 1
+    list_of_ids = process.data_miner.mine_ids_sorted()
+    list_of_predictions = []
+    for i in range(0, len(record), 3):
+        prediction = m.PredictedMeasurement(record[i], record[i + 1], record[i + 2], predicted_date,
+                                            list_of_ids[i // 3])
+        list_of_predictions.append(prediction)
+    with app.app_context():
+        m.add_predicted_measurements(list_of_predictions)
+
+
+def save_record_week(record: np.array, date: datetime):
+    print(record.shape)
+    list_of_ids = process.data_miner.mine_ids_sorted()
+    print(list_of_ids)
+    list_of_predictions = []
+    for i in range(12):
+        predicted_date = date + timedelta(hours=i)
+        for j in range(i*150,(i+1)*150 , 3):
+            prediction = m.PredictedMeasurement(record[0,146+i,j], record[0,146+i,j + 1], record[0,146+i,j + 2], predicted_date,
+                                                list_of_ids[(j - i * 150) // 3])
+            list_of_predictions.append(prediction)
+    print(len(list_of_predictions))
+    with app.app_context():
+        m.add_predicted_measurements(list_of_predictions)
+
+def _predict_for_test(idx:int = None):
+    data, weather = fetch_data()
+    batched_data, batched_weather = batch_data_week(data, weather, idx)
+
+    model = load_gustaw()
+    out = model.predict([batched_data, batched_weather])
+    save_gustaw(model)
+
+    return out

backend/kst_app/engine/predictor_test.py

@@ -0,0 +1,22 @@
+import unittest
+from kst_app.data_storage.models import get_all_predicted_measurements
+from kst_app.engine.predictor import predict_12h, _predict_for_test
+from kst_app import app
+import time
+
+
+class TestPredictior(unittest.TestCase):
+    def test_prediction_12h(self):
+        with app.app_context():
+            predict_12h()
+            measurements = get_all_predicted_measurements()
+            print(measurements)
+            self.assertEqual(12*50, len(measurements))
+
+    def test_preds(self):
+        with app.app_context():
+            self.assertEqual(12 * 50, len(get_all_predicted_measurements()))
+
+    def test_prediction_on_data(self):
+        with app.app_context():
+            out = _predict_for_test(100)