diff --git a/backend/kst_app/__init__.py b/backend/kst_app/__init__.py index 379126f..458eb7c 100644 --- a/backend/kst_app/__init__.py +++ b/backend/kst_app/__init__.py @@ -13,3 +13,4 @@ with app.app_context(): db.create_all() + diff --git a/backend/kst_app/data.pkl b/backend/kst_app/data.pkl new file mode 100644 index 0000000..8809502 Binary files /dev/null and b/backend/kst_app/data.pkl differ diff --git a/backend/kst_app/data_dict.pickle b/backend/kst_app/data_dict.pickle new file mode 100644 index 0000000..1393ad8 Binary files /dev/null and b/backend/kst_app/data_dict.pickle differ diff --git a/backend/kst_app/data_processing/__init__.py b/backend/kst_app/data_processing/__init__.py index e69de29..410e46e 100755 --- a/backend/kst_app/data_processing/__init__.py +++ b/backend/kst_app/data_processing/__init__.py @@ -0,0 +1,2 @@ +from .data_miner import mine_ids_sorted +from .batcher import batch_sensor_data_day, batch_auxiliary_data_day, batch_sensor_data_week, batch_auxiliary_data_week diff --git a/backend/kst_app/data_processing/batcher.py b/backend/kst_app/data_processing/batcher.py new file mode 100644 index 0000000..a5aea5d --- /dev/null +++ b/backend/kst_app/data_processing/batcher.py @@ -0,0 +1,90 @@ +from typing import List +import numpy as np +from kst_app.data_storage.models import Measurement, WeatherMeasurement + + +def batch_sensor_data_day(measurment_list: List[Measurement]) -> np.array: + assert len(measurment_list) >= 24 * 50, 'not enough data' # The number of measurments in one day + last_24 = measurment_list[:24 * 50] + + batched_sensors = np.zeros([1, 24, 150]) + counter_sensors = 0 + counter_hours = 0 + for measurment in last_24: + batched_sensors[0, counter_hours, counter_sensors * 3:(counter_sensors + 1) * 3] = [measurment.pm1, + measurment.pm25, + measurment.pm10] + counter_sensors += 1 + if counter_sensors == 50: + counter_sensors = 0 + counter_hours += 1 + + return batched_sensors + +def batch_sensor_data_week(measurment_list: List[Measurement], idx:int = None): + if idx is None: + last_week = measurment_list[-24*50*7:] + else: + last_week = measurment_list[idx:idx+24*50*7] + assert len(last_week) >= 24 * 50 * 7, 'not enough data' + + batched_sensors = np.zeros([1, 24 * 7, 150]) + counter_sensors = 0 + counter_hours = 0 + for measurment in last_week: + batched_sensors[0, counter_hours, counter_sensors * 3:(counter_sensors + 1) * 3] = [measurment.pm1, + measurment.pm25, + measurment.pm10] + counter_sensors += 1 + if counter_sensors == 50: + counter_sensors = 0 + counter_hours += 1 + + return batched_sensors + + + + +def batch_auxiliary_data_day(measurment_list: List[WeatherMeasurement]) -> np.array: + assert len(measurment_list) >= 24, 'not enough weather data' + last_24 = measurment_list[0:24] + + batched_weather = np.zeros((1, 24, 10)) + + for i, measurment in enumerate(last_24): + batched_weather[0, i] = [measurment.rainLevel, + measurment.temperature, + measurment.humidity, + measurment.pressure, + measurment.windDirection, + measurment.windSpeed, + measurment.date.hour, + measurment.date.day, + measurment.date.month, + measurment.date.year] + + return batched_weather + +def batch_auxiliary_data_week(measurment_list: List[WeatherMeasurement], idx:int = None) -> np.array: + if idx is None: + last_week = measurment_list[-24*7:] + else: + last_week = measurment_list[idx:idx+24*7] + + assert len(last_week) >= 24 * 7, 'not enough weather data' + + batched_weather = np.zeros((1, 24*7, 10)) + + for i, measurment in enumerate(last_week): + batched_weather[0, i] = [measurment.rainLevel, + measurment.temperature, + measurment.humidity, + measurment.pressure, + measurment.windDirection, + measurment.windSpeed, + measurment.date.hour, + measurment.date.day, + measurment.date.month, + measurment.date.year] + + return batched_weather diff --git a/backend/kst_app/data_processing/data_miner.py b/backend/kst_app/data_processing/data_miner.py new file mode 100644 index 0000000..15877f0 --- /dev/null +++ b/backend/kst_app/data_processing/data_miner.py @@ -0,0 +1,8 @@ +from typing import List +from kst_app.data_storage.models import get_all_sensors + + +def mine_ids_sorted() -> List[int]: + sensors = get_all_sensors() + sensors = sorted(sensors, key=lambda sensor: sensor.id) + return [sensor.id for sensor in sensors] diff --git a/backend/kst_app/data_storage/config.py b/backend/kst_app/data_storage/config.py new file mode 100644 index 0000000..bf91830 --- /dev/null +++ b/backend/kst_app/data_storage/config.py @@ -0,0 +1,2 @@ +DEBUG = True # Turns on debugging features in Flask +SQLALCHEMY_DATABASE_URI = "mysql+mysqldb://root:secret_password@localhost:3306/kst_db" diff --git a/backend/kst_app/data_storage/models.py b/backend/kst_app/data_storage/models.py index 387b6a1..6d519c6 100644 --- a/backend/kst_app/data_storage/models.py +++ b/backend/kst_app/data_storage/models.py @@ -11,6 +11,7 @@ class Sensor(db.Model): + __table_args__ = {'extend_existing': True} id = db.Column(db.Integer, primary_key=True) longitude = db.Column(db.Float(precision=10)) latitude = db.Column(db.Float(precision=10)) @@ -81,6 +82,7 @@ def update_sensor(sensor_id: int, longitude: float, latitude: float): class Measurement(db.Model): + __table_args__ = {'extend_existing': True} id = db.Column(db.Integer, primary_key=True) pm1 = db.Column(db.Float(precision=10)) pm25 = db.Column(db.Float(precision=10)) @@ -135,7 +137,13 @@ def get_measurement(measurement_id: int) -> Measurement: def get_all_measurements() -> List[Measurement]: - return Measurement.query.all() + return Measurement.query.order_by(Measurement.date, Measurement.sensor_id).all() + + +def delete_all_measurements() -> int: + deleted = Measurement.query.delete() + db.session.commit() + return deleted def delete_measurement(measurement_id: int) -> Measurement: @@ -176,6 +184,7 @@ def update_measurement(measurement_id: int, pm1: float, pm25: float, pm10: float class PredictedMeasurement(db.Model): + __table_args__ = {'extend_existing': True} id = db.Column(db.Integer, primary_key=True) pm1 = db.Column(db.Float(precision=10)) pm25 = db.Column(db.Float(precision=10)) @@ -221,7 +230,7 @@ def get_predicted_measurement(measurement_id: int) -> PredictedMeasurement: def get_all_predicted_measurements() -> List[PredictedMeasurement]: - return PredictedMeasurement.query.all() + return PredictedMeasurement.query.order_by(PredictedMeasurement.date, PredictedMeasurement.sensor_id).all() def delete_predicted_measurement(measurement_id: int) -> PredictedMeasurement: @@ -231,6 +240,12 @@ def delete_predicted_measurement(measurement_id: int) -> PredictedMeasurement: return predicted_measurement +def delete_all_predicted_measurements() -> List[PredictedMeasurement]: + deleted = PredictedMeasurement.query.delete() + db.session.commit() + return deleted + + def update_predicted_measurement(measurement_id: int, pm1: float, pm25: float, pm10: float, date: datetime, sensor_id: int) -> PredictedMeasurement: measurement = PredictedMeasurement.query.filter_by(id=measurement_id).one() @@ -254,6 +269,7 @@ def update_predicted_measurement(measurement_id: int, pm1: float, pm25: float, p class WeatherMeasurement(db.Model): + __table_args__ = {'extend_existing': True} id = db.Column(db.Integer, primary_key=True) temperature = db.Column(db.Float(precision=10)) humidity = db.Column(db.Float(precision=10)) @@ -310,7 +326,9 @@ def get_weather_measurement(measurement_id: int) -> WeatherMeasurement: def get_all_weather_measurements() -> List[WeatherMeasurement]: - return WeatherMeasurement.query.all() + deleted = WeatherMeasurement.query.order_by(WeatherMeasurement.date).all() + db.session.commit() + return deleted def delete_weather_measurement(measurement_id: int) -> WeatherMeasurement: @@ -320,6 +338,10 @@ def delete_weather_measurement(measurement_id: int) -> WeatherMeasurement: return weather_measurement +def delete_all_weather_measurement() -> int: + return WeatherMeasurement.query.delete() + + def update_weather_measurement(measurement_id: int, temperature: float, humidity: float, pressure: float, windSpeed: float, windDirection: float, rainLevel: float, date: datetime, sensor_id: int) -> WeatherMeasurement: diff --git a/backend/kst_app/data_storage/models_test.py b/backend/kst_app/data_storage/models_test.py new file mode 100644 index 0000000..4e04674 --- /dev/null +++ b/backend/kst_app/data_storage/models_test.py @@ -0,0 +1,16 @@ +import unittest +import pickle +from .models import Measurement, add_measurements + +class TestModels(unittest.TestCase): + def add_measurements_test(self): + with open('C:\\Users\\bartw\\OneDrive\\Pulpit\\KST\\backend\\kst_app\\data.pkl', 'rb') as port: + data = pickle.load(port) + + list_of_measurments = [] + for key, value in data.items(): + for id, info in value.items(): + measurment = Measurement(info[0], info[1], info[2], 0, 0, 0, key, id) + list_of_measurments.append(measurment) + + add_measurements(list_of_measurments) diff --git a/backend/kst_app/engine/TFModel.py b/backend/kst_app/engine/TFModel.py new file mode 100644 index 0000000..a0a2807 --- /dev/null +++ b/backend/kst_app/engine/TFModel.py @@ -0,0 +1,25 @@ +import tensorflow as tf + +model = tf.keras.Model + + +def load_model(): + global model + model = tf.keras.models.load_model('model1h.h5') + + +def save_model(): + model.save('model1h.h5') + tf.keras.backend.clear_session() + + +def load_gustaw(): + def alst_time_step_mse(Y_true, Y_pred): + return tf.keras.metrics.mean_squared_error(Y_true[:, -1], Y_pred[:, -1]) + + return tf.keras.models.load_model('gustaw.h5',custom_objects={"alst_time_step_mse": alst_time_step_mse}) + + +def save_gustaw(gustaw: tf.keras.Model): + gustaw.save('gustaw.h5') + tf.keras.backend.clear_session() diff --git a/backend/kst_app/engine/__init__.py b/backend/kst_app/engine/__init__.py index e69de29..8d5f204 100755 --- a/backend/kst_app/engine/__init__.py +++ b/backend/kst_app/engine/__init__.py @@ -0,0 +1 @@ +from .predictor import predict_hour \ No newline at end of file diff --git a/backend/kst_app/engine/model1h.h5 b/backend/kst_app/engine/model1h.h5 new file mode 100644 index 0000000..7e5df0b Binary files /dev/null and b/backend/kst_app/engine/model1h.h5 differ diff --git a/backend/kst_app/engine/model_architecture.py b/backend/kst_app/engine/model_architecture.py new file mode 100644 index 0000000..696deac --- /dev/null +++ b/backend/kst_app/engine/model_architecture.py @@ -0,0 +1,22 @@ +import tensorflow as tf + +def create_clean_model_hour(): + input_rec = tf.keras.layers.Input((24, 150)) + norm = tf.keras.layers.Normalization()(input_rec) + LSTM1 = tf.keras.layers.LSTM(800, return_sequences=True)(norm) + LSTM2 = tf.keras.layers.LSTM(800, dropout=.1)(LSTM1) + + inputs_aux = tf.keras.layers.Input((24, 10)) + flatten = tf.keras.layers.Flatten()(inputs_aux) + pre_preprocess = tf.keras.layers.Dense(256, activation='elu', kernel_regularizer='l2')(flatten) + pre_preprocess = tf.keras.layers.Dense(256, activation='elu', kernel_regularizer='l2')(pre_preprocess) + preprocess = tf.keras.layers.Dense(128, activation='elu', kernel_regularizer='l2')(pre_preprocess) + + combine = tf.keras.layers.Concatenate()([preprocess, LSTM2]) + broad_layer = tf.keras.layers.Dense(300, activation='elu', kernel_regularizer='l2')(combine) + out = tf.keras.layers.Dense(150, activation='linear')(broad_layer) + + model = tf.keras.Model([input_rec, inputs_aux], out) + model.compile(loss='mse', optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001), metrics=['mse']) + + return model \ No newline at end of file diff --git a/backend/kst_app/engine/predictor.py b/backend/kst_app/engine/predictor.py new file mode 100644 index 0000000..183ecaf --- /dev/null +++ b/backend/kst_app/engine/predictor.py @@ -0,0 +1,88 @@ +from kst_app.engine.TFModel import model, load_model, save_model, load_gustaw, save_gustaw +import kst_app.data_processing as process +from kst_app.data_storage import models as m +from kst_app import app +from typing import List, Tuple +import numpy as np +from datetime import datetime, timedelta + + +def predict_hour(): + data, weather = fetch_data() + batched_data, batched_weather = batch_data_day(data, weather) + + last_measurment_date = weather[0].date + + load_model() + out = model.predict([batched_data, batched_weather]) + save_model() + + save_record(out, last_measurment_date) + + +def predict_12h(): + delete_previous_predictions() + + data, weather = fetch_data() + batched_data, batched_weather = batch_data_week(data, weather) + + last_measurment_date = weather[-1].date + + model = load_gustaw() + out = model.predict([batched_data, batched_weather]) + save_gustaw(model) + + save_record_week(out, last_measurment_date) + + +def fetch_data() -> Tuple[List[m.Measurement], List[m.WeatherMeasurement]]: + return m.get_all_measurements(), m.get_all_weather_measurements() + +def delete_previous_predictions(): + m.delete_all_predicted_measurements() + + +def batch_data_day(data: List[m.Measurement], weather: List[m.WeatherMeasurement]) -> Tuple[np.array, np.array]: + return process.batch_sensor_data_day(data), process.batch_auxiliary_data_day(weather) + + +def batch_data_week(data: List[m.Measurement], weather: List[m.WeatherMeasurement],idx:int = None) -> Tuple[np.array, np.array]: + return process.batch_sensor_data_week(data,idx), process.batch_auxiliary_data_week(weather,idx) + + +def save_record(record: List[float], date: datetime): + predicted_date = date.hour + 1 + list_of_ids = process.data_miner.mine_ids_sorted() + list_of_predictions = [] + for i in range(0, len(record), 3): + prediction = m.PredictedMeasurement(record[i], record[i + 1], record[i + 2], predicted_date, + list_of_ids[i // 3]) + list_of_predictions.append(prediction) + with app.app_context(): + m.add_predicted_measurements(list_of_predictions) + + +def save_record_week(record: np.array, date: datetime): + print(record.shape) + list_of_ids = process.data_miner.mine_ids_sorted() + print(list_of_ids) + list_of_predictions = [] + for i in range(12): + predicted_date = date + timedelta(hours=i) + for j in range(i*150,(i+1)*150 , 3): + prediction = m.PredictedMeasurement(record[0,146+i,j], record[0,146+i,j + 1], record[0,146+i,j + 2], predicted_date, + list_of_ids[(j - i * 150) // 3]) + list_of_predictions.append(prediction) + print(len(list_of_predictions)) + with app.app_context(): + m.add_predicted_measurements(list_of_predictions) + +def _predict_for_test(idx:int = None): + data, weather = fetch_data() + batched_data, batched_weather = batch_data_week(data, weather, idx) + + model = load_gustaw() + out = model.predict([batched_data, batched_weather]) + save_gustaw(model) + + return out diff --git a/backend/kst_app/engine/predictor_test.py b/backend/kst_app/engine/predictor_test.py new file mode 100644 index 0000000..a6056a7 --- /dev/null +++ b/backend/kst_app/engine/predictor_test.py @@ -0,0 +1,22 @@ +import unittest +from kst_app.data_storage.models import get_all_predicted_measurements +from kst_app.engine.predictor import predict_12h, _predict_for_test +from kst_app import app +import time + + +class TestPredictior(unittest.TestCase): + def test_prediction_12h(self): + with app.app_context(): + predict_12h() + measurements = get_all_predicted_measurements() + print(measurements) + self.assertEqual(12*50, len(measurements)) + + def test_preds(self): + with app.app_context(): + self.assertEqual(12 * 50, len(get_all_predicted_measurements())) + + def test_prediction_on_data(self): + with app.app_context(): + out = _predict_for_test(100) diff --git a/backend/kst_app/sensors.pkl b/backend/kst_app/sensors.pkl new file mode 100644 index 0000000..bff99c2 Binary files /dev/null and b/backend/kst_app/sensors.pkl differ diff --git a/backend/kst_app/weather.pickle b/backend/kst_app/weather.pickle new file mode 100644 index 0000000..3d1a9d2 Binary files /dev/null and b/backend/kst_app/weather.pickle differ diff --git a/backend/kst_app/weather.pkl b/backend/kst_app/weather.pkl new file mode 100644 index 0000000..d3776b3 Binary files /dev/null and b/backend/kst_app/weather.pkl differ diff --git a/backend/kst_app/weather_DF.pickle b/backend/kst_app/weather_DF.pickle new file mode 100644 index 0000000..cb7aa09 Binary files /dev/null and b/backend/kst_app/weather_DF.pickle differ diff --git a/backend/model_test.py b/backend/model_test.py new file mode 100644 index 0000000..ce81548 --- /dev/null +++ b/backend/model_test.py @@ -0,0 +1,33 @@ +import unittest +import numpy as np +from kst_app.engine.model_architecture import create_clean_model_hour +from kst_app import app + + +class ModelTest(unittest.TestCase): + def test_learning_const(self): + data = np.ones((200,24,150)) + target = np.ones((200,150)) + weather = np.ones((200,24,10)) + + model = create_clean_model_hour() + model.fit([data[:160],weather[:160]],target[:160], epochs=4) + + _, metrics = model.evaluate([data[160:],weather[160:]],target[160:]) + self.assertGreater(1,metrics,'Not good enough const') + + def test_learning_linear(self): + data = np.zeros((200,24,150)) + target = np.zeros((200,150)) + weather = np.ones((200,24,10)) + + for i in range(200): + data[i] = np.ones((24,150)) * i + target[i] = i + weather[i] = np.ones((24,10)) * i + + model = create_clean_model_hour() + model.fit([data[:160], weather[:160]], target[:160], epochs=4) + + _, metrics = model.evaluate([data[160:], weather[160:]], target[160:]) + self.assertGreater(1, metrics, 'Not good enough linear') diff --git a/backend/requirements_new.txt b/backend/requirements_new.txt new file mode 100644 index 0000000..6bf2558 Binary files /dev/null and b/backend/requirements_new.txt differ diff --git a/backend/requirments_new.txt b/backend/requirments_new.txt new file mode 100644 index 0000000..a3f7986 Binary files /dev/null and b/backend/requirments_new.txt differ diff --git a/backend/test.py b/backend/test.py new file mode 100644 index 0000000..05e6402 --- /dev/null +++ b/backend/test.py @@ -0,0 +1,61 @@ +from kst_app.data_storage.models import Sensor, add_sensors, Measurement, add_measurements, delete_all_measurements, delete_all_weather_measurement,WeatherMeasurement, add_weather_measurements, get_all_measurements, get_all_weather_measurements +from kst_app import app +import pickle +import pandas + +if __name__ == '__main__': + + # with app.app_context(): + # print(delete_all_measurements()) + + # with app.app_context(): + # print(delete_all_weather_measurement()) + + with open('C:\\Users\\bartw\\OneDrive\\Pulpit\\KST\\backend\\kst_app\\data_dict.pickle', 'rb') as port: + data = pickle.load(port) + + with open('C:\\Users\\bartw\\OneDrive\\Pulpit\\KST\\backend\\kst_app\\weather_DF.pickle', 'rb') as port: + weather = pickle.load(port) + + # with open('C:\\Users\\bartw\\OneDrive\\Pulpit\\KST\\backend\\kst_app\\sensors.pkl','rb') as port: + # sensors = pickle.load(port) + + # list_of_sensors = [] + # for key, value in sensors.items(): + # sensor = Sensor(value['x'], value['y']) + # sensor.id = key + # list_of_sensors.append(sensor) + # print('done') + + # with app.app_context(): + # add_sensors(list_of_sensors) + # print('done') + + + # list_of_measurments = [] + # for key, value in data.items(): + # for id, info in value.items(): + # measurment = Measurement(info[0], info[1], info[2], 0, 0, 0, key, id) + # list_of_measurments.append(measurment) + # print('done') + # + # with app.app_context(): + # add_measurements(list_of_measurments) + # print('done') + + # with app.app_context(): + # sensor = Sensor(0,0) + # sensor.id = 1 + # add_sensors([sensor]) + + + # list_of_weather = [] + # weather = weather.drop(['hour','day','month','year'], axis=1).to_numpy() + # for w in weather: + # list_of_weather.append(WeatherMeasurement(temperature=w[1], humidity=w[2],pressure=w[3], windSpeed=[5], windDirection=w[4],rainLevel=w[6], date=w[0], sensor_id=1)) + # + # with app.app_context(): + # add_weather_measurements(list_of_weather) + # print('done') + +