From f7d74c101b64be0167a906345f3814ba68297976 Mon Sep 17 00:00:00 2001 From: Lucie Delestre Date: Fri, 23 May 2025 16:05:02 +0200 Subject: [PATCH 01/12] Ajout route API pour les horaires de bus C6/75 --- handlers/naolib.go | 37 +++++++++++++++++++++++ main.go | 3 ++ models/naolib.go | 24 +++++++++++++++ routes/naolib_routes.go | 16 ++++++++++ services/naolib.go | 65 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 145 insertions(+) create mode 100644 handlers/naolib.go create mode 100644 models/naolib.go create mode 100644 routes/naolib_routes.go create mode 100644 services/naolib.go diff --git a/handlers/naolib.go b/handlers/naolib.go new file mode 100644 index 0000000..cd06dd3 --- /dev/null +++ b/handlers/naolib.go @@ -0,0 +1,37 @@ +package handlers + +import ( + "github.com/gofiber/fiber/v2" + "github.com/plugimt/transat-backend/services" + "github.com/plugimt/transat-backend/utils" +) + +type NaolibHandler struct { + service *services.NaolibService +} + +func NewNaolibHandler(service *services.NaolibService) *NaolibHandler { + return &NaolibHandler{ + service: service, + } +} + +func (h *NaolibHandler) GetNextDeparturesChantrerie(c *fiber.Ctx) error { + departuresC6, err := h.service.GetNextDepartures("CTRE2") + if err != nil { + utils.LogMessage(utils.LevelError, "Error fetching departures: "+err.Error()) + return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ + "error": "Failed to fetch departure information", + }) + } + departures75, err := h.service.GetNextDepartures("CTRE4") + if err != nil { + utils.LogMessage(utils.LevelError, "Error fetching departures: "+err.Error()) + } + + departures := append(departuresC6, departures75...) + return c.JSON(fiber.Map{ + "success": true, + "data": departures, + }) +} diff --git a/main.go b/main.go index 36fc7c5..4f0490a 100644 --- a/main.go +++ b/main.go @@ -119,6 +119,8 @@ func main() { // Initialize Handlers that need explicit instantiation (e.g., for Cron) restHandler := restaurantHandler.NewRestaurantHandler(db, translationService, notificationService) + naolibService := services.NewNaolibService(30 * time.Second) + // Initialize Weather Service and Handler weatherService, err := services.NewWeatherService() if err != nil { @@ -197,6 +199,7 @@ func main() { routes.SetupWashingMachineRoutes(api) // Setup washing machine routes routes.SetupWeatherRoutes(api, weatherHandler) // Setup weather routes routes.SetupNotificationRoutes(api, db, notificationService) // Setup notification test routes + routes.SetupNaolibRoutes(api, naolibService) app.Get("/health", func(c *fiber.Ctx) error { return c.SendString("OK") diff --git a/models/naolib.go b/models/naolib.go new file mode 100644 index 0000000..b0450fe --- /dev/null +++ b/models/naolib.go @@ -0,0 +1,24 @@ +package models + +// Departure représente un départ de bus +type Departure struct { + Sens int `json:"sens"` + Terminus string `json:"terminus"` + InfoTrafic bool `json:"infotrafic"` + Temps string `json:"temps"` + DernierDepart string `json:"dernierDepart"` + TempsReel string `json:"tempsReel"` + Ligne Ligne `json:"ligne"` + Arret Arret `json:"arret"` +} + +// Ligne représente les informations sur la ligne de bus +type Ligne struct { + NumLigne string `json:"numLigne"` + TypeLigne int `json:"typeLigne"` +} + +// Arret représente les informations sur l'arrêt de bus +type Arret struct { + CodeArret string `json:"codeArret"` +} diff --git a/routes/naolib_routes.go b/routes/naolib_routes.go new file mode 100644 index 0000000..7eba9af --- /dev/null +++ b/routes/naolib_routes.go @@ -0,0 +1,16 @@ +package routes + +import ( + "github.com/gofiber/fiber/v2" + "github.com/plugimt/transat-backend/handlers" + "github.com/plugimt/transat-backend/services" +) + +func SetupNaolibRoutes(router fiber.Router, naolibService *services.NaolibService) { + // Groupe de routes pour Naolib + naolib := router.Group("/naolib") + handler := handlers.NewNaolibHandler(naolibService) + + // Route pour obtenir les prochains départs + naolib.Get("/departures/chantrerie", handler.GetNextDeparturesChantrerie) +} diff --git a/services/naolib.go b/services/naolib.go new file mode 100644 index 0000000..b30c2d7 --- /dev/null +++ b/services/naolib.go @@ -0,0 +1,65 @@ +package services + +import ( + "encoding/json" + "fmt" + "net/http" + "sync" + "time" + + "github.com/plugimt/transat-backend/models" + "github.com/plugimt/transat-backend/utils" +) + +var httpClient = &http.Client{ + Timeout: 10 * time.Second, +} + +type NaolibService struct { + mu sync.Mutex + cache map[string][]models.Departure + cacheTime time.Duration + lastUpdate map[string]time.Time +} + +func NewNaolibService(refreshTime time.Duration) *NaolibService { + return &NaolibService{ + cache: make(map[string][]models.Departure), + cacheTime: refreshTime, + lastUpdate: make(map[string]time.Time), + } +} + +func (s *NaolibService) GetNextDepartures(stopID string) ([]models.Departure, error) { + s.mu.Lock() + if lastUpdateTime, updateExists := s.lastUpdate[stopID]; updateExists { + if departures, ok := s.cache[stopID]; ok && time.Since(lastUpdateTime) < s.cacheTime { + s.mu.Unlock() + utils.LogMessage(utils.LevelInfo, fmt.Sprintf("Using cached departures for %s", stopID)) + return departures, nil + } + } + + url := fmt.Sprintf("https://open.tan.fr/ewp/tempsattentelieu.json/%s/2", stopID) + + resp, err := httpClient.Get(url) + if err != nil { + return nil, fmt.Errorf("erreur lors de la requête HTTP: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("erreur HTTP: code %d", resp.StatusCode) + } + + var departuresData []models.Departure + if err := json.NewDecoder(resp.Body).Decode(&departuresData); err != nil { + return nil, fmt.Errorf("erreur lors du décodage JSON: %v", err) + } + + s.cache[stopID] = departuresData + s.lastUpdate[stopID] = time.Now() + s.mu.Unlock() + + return departuresData, nil +} From 9442430065fa0da98fc69c28ad9e2301d274eedf Mon Sep 17 00:00:00 2001 From: Lucie Delestre Date: Fri, 23 May 2025 20:54:32 +0200 Subject: [PATCH 02/12] Import NETEX dans la DB par POST --- db/migrations/00009_add_netex_data.sql | 50 +++++++++++ handlers/naolib.go | 93 ++++++++++++++++++- main.go | 2 +- models/netex.go | 101 +++++++++++++++++++++ routes/naolib_routes.go | 9 +- services/netex/decode.go | 118 +++++++++++++++++++++++++ 6 files changed, 369 insertions(+), 4 deletions(-) create mode 100644 db/migrations/00009_add_netex_data.sql create mode 100644 models/netex.go create mode 100644 services/netex/decode.go diff --git a/db/migrations/00009_add_netex_data.sql b/db/migrations/00009_add_netex_data.sql new file mode 100644 index 0000000..018fc9c --- /dev/null +++ b/db/migrations/00009_add_netex_data.sql @@ -0,0 +1,50 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE NETEX_StopPlace ( + id VARCHAR(255) PRIMARY KEY, + modification VARCHAR(50), + created_timestamp TIMESTAMPTZ, + changed_timestamp TIMESTAMPTZ, + valid_from_date TIMESTAMPTZ, + valid_to_date TIMESTAMPTZ, + name VARCHAR(255), + name_lang VARCHAR(10), + longitude DECIMAL(9,6), + latitude DECIMAL(8,6), + transport_mode VARCHAR(50), + other_transport_modes TEXT, + stop_place_type VARCHAR(50), + weighting VARCHAR(50), + UNIQUE (id) +); + +CREATE TABLE NETEX_Quay ( + id VARCHAR(255) PRIMARY KEY, + changed_timestamp TIMESTAMPTZ, + name VARCHAR(255), + name_lang VARCHAR(10), + longitude DECIMAL(9,6), + latitude DECIMAL(8,6), + postal_region VARCHAR(50), + site_ref_stopplace_id VARCHAR(255) REFERENCES NETEX_StopPlace(id) ON DELETE SET NULL, + transport_mode VARCHAR(50), + UNIQUE (id) +); + +CREATE TABLE NETEX_StopPlace_QuayRef ( + stop_place_id VARCHAR(255) REFERENCES NETEX_StopPlace(id) ON DELETE CASCADE, + quay_id VARCHAR(255) REFERENCES NETEX_Quay(id) ON DELETE CASCADE, + quay_ref_version VARCHAR(50), + PRIMARY KEY (stop_place_id, quay_id), + UNIQUE (stop_place_id, quay_id) +); + + +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE NETEX_StopPlace; +DROP TABLE NETEX_Quay; +DROP TABLE NETEX_StopPlace_QuayRef; +-- +goose StatementEnd diff --git a/handlers/naolib.go b/handlers/naolib.go index cd06dd3..e3dd45c 100644 --- a/handlers/naolib.go +++ b/handlers/naolib.go @@ -1,18 +1,24 @@ package handlers import ( + "database/sql" + "os" + "github.com/gofiber/fiber/v2" "github.com/plugimt/transat-backend/services" + "github.com/plugimt/transat-backend/services/netex" "github.com/plugimt/transat-backend/utils" ) type NaolibHandler struct { service *services.NaolibService + db *sql.DB } -func NewNaolibHandler(service *services.NaolibService) *NaolibHandler { +func NewNaolibHandler(service *services.NaolibService, db *sql.DB) *NaolibHandler { return &NaolibHandler{ service: service, + db: db, } } @@ -35,3 +41,88 @@ func (h *NaolibHandler) GetNextDeparturesChantrerie(c *fiber.Ctx) error { "data": departures, }) } + +func (h *NaolibHandler) ImportNetexData(c *fiber.Ctx) error { + var body struct { + Url string `json:"url"` + } + + if err := c.BodyParser(&body); err != nil { + return c.Status(fiber.StatusBadRequest).SendString(err.Error()) + } + + url := body.Url + + if url == "" { + return c.Status(fiber.StatusBadRequest).SendString("URL is required") + } + + fileName, err := netex.DownloadAndExtractIfNeeded(url) + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + defer os.Remove(fileName) + + netexData, err := netex.DecodeNetexData(fileName) + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + + stopPlaces := netexData.DataObjects.GeneralFrame.Members.StopPlaces + quays := netexData.DataObjects.GeneralFrame.Members.Quays + + // on crée une transaction et on supprime toutes les données existantes, avant de les insérer + tx, err := h.db.Begin() + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + defer tx.Rollback() + + _, err = tx.Exec("DELETE FROM NETEX_StopPlace") + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + + _, err = tx.Exec("DELETE FROM NETEX_Quay") + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + + _, err = tx.Exec("DELETE FROM NETEX_StopPlace_QuayRef") + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + + for _, stopPlace := range stopPlaces { + _, err := tx.Exec("INSERT INTO NETEX_StopPlace (id, modification, name, longitude, latitude, transport_mode, other_transport_modes, stop_place_type, weighting) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", stopPlace.ID, stopPlace.Modification, stopPlace.Name, stopPlace.Centroid.Location.Longitude, stopPlace.Centroid.Location.Latitude, stopPlace.TransportMode, stopPlace.OtherTransportModes, stopPlace.StopPlaceType, stopPlace.Weighting) + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + } + + for _, quay := range quays { + _, err := tx.Exec("INSERT INTO NETEX_Quay (id, name, longitude, latitude, site_ref_stopplace_id, transport_mode) VALUES ($1, $2, $3, $4, $5, $6)", quay.ID, quay.Name, quay.Centroid.Location.Longitude, quay.Centroid.Location.Latitude, quay.SiteRef.Ref, quay.TransportMode) + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + } + + // now, populate the NETEX_StopPlace_QuayRef link table + for _, stopPlace := range stopPlaces { + for _, quayRef := range stopPlace.QuayRefs { + _, err := tx.Exec("INSERT INTO NETEX_StopPlace_QuayRef (stop_place_id, quay_id, quay_ref_version) VALUES ($1, $2, $3)", stopPlace.ID, quayRef.Ref, quayRef.Version) + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + } + } + + err = tx.Commit() + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + + return c.JSON(map[string]interface{}{ + "success": true, + }) +} diff --git a/main.go b/main.go index 4f0490a..ab9f30d 100644 --- a/main.go +++ b/main.go @@ -199,7 +199,7 @@ func main() { routes.SetupWashingMachineRoutes(api) // Setup washing machine routes routes.SetupWeatherRoutes(api, weatherHandler) // Setup weather routes routes.SetupNotificationRoutes(api, db, notificationService) // Setup notification test routes - routes.SetupNaolibRoutes(api, naolibService) + routes.SetupNaolibRoutes(api, naolibService, db) app.Get("/health", func(c *fiber.Ctx) error { return c.SendString("OK") diff --git a/models/netex.go b/models/netex.go new file mode 100644 index 0000000..c5eed4f --- /dev/null +++ b/models/netex.go @@ -0,0 +1,101 @@ +package models + +import "encoding/xml" + +type NETEXStopsFile struct { + XMLName xml.Name `xml:"PublicationDelivery"` + PublicationDelivery PublicationDelivery `xml:"PublicationDelivery"` +} + +type PublicationDelivery struct { + XMLName xml.Name `xml:"PublicationDelivery"` + PublicationTimestamp string `xml:"PublicationTimestamp"` + ParticipantRef string `xml:"ParticipantRef"` + DataObjects DataObjects `xml:"dataObjects"` + Version string `xml:"version,attr"` + Xmlns string `xml:"xmlns,attr"` + XmlnsAcbs string `xml:"xmlns:acbs,attr"` + XmlnsGML string `xml:"xmlns:gml,attr"` + XmlnsIfopt string `xml:"xmlns:ifopt,attr"` + XmlnsSiri string `xml:"xmlns:siri,attr"` + XmlnsXSD string `xml:"xmlns:xsd,attr"` + XmlnsXsi string `xml:"xmlns:xsi,attr"` +} + +type DataObjects struct { + GeneralFrame GeneralFrame `xml:"GeneralFrame"` +} + +type GeneralFrame struct { + ID string `xml:"id,attr"` + Modification string `xml:"modification,attr"` + Version string `xml:"version,attr"` + TypeOfFrameRef TypeOfFrameRef `xml:"TypeOfFrameRef"` + Members Members `xml:"members"` +} + +type TypeOfFrameRef struct { + Ref string `xml:"ref,attr"` + Text string `xml:",chardata"` +} + +type Members struct { + Quays []Quay `xml:"Quay"` + StopPlaces []StopPlace `xml:"StopPlace"` +} + +type Quay struct { + ID string `xml:"id,attr"` + Modification string `xml:"modification,attr"` + Version string `xml:"version,attr"` + KeyList []KeyValueHolder `xml:"keyList>KeyValue"` + Name string `xml:"Name"` + Centroid Centroid `xml:"Centroid"` + PostalAddress PostalAddress `xml:"PostalAddress"` + SiteRef SiteRef `xml:"SiteRef"` + TransportMode string `xml:"TransportMode"` +} + +type StopPlace struct { + ID string `xml:"id,attr"` + Modification string `xml:"modification,attr"` + Version string `xml:"version,attr"` + Name string `xml:"Name"` + Centroid Centroid `xml:"Centroid"` + TransportMode string `xml:"TransportMode"` + OtherTransportModes string `xml:"OtherTransportModes"` + StopPlaceType string `xml:"StopPlaceType"` + Weighting string `xml:"Weighting"` + QuayRefs []QuayRef `xml:"quays>QuayRef"` + KeyList []KeyValueHolder `xml:"keyList>KeyValue"` +} + +type KeyValueHolder struct { + Key string `xml:"Key"` + Value string `xml:"Value"` +} + +type QuayRef struct { + Ref string `xml:"ref,attr"` + Version string `xml:"version,attr"` +} + +type Centroid struct { + Location Location `xml:"Location"` +} + +type Location struct { + Longitude string `xml:"Longitude"` + Latitude string `xml:"Latitude"` +} + +type PostalAddress struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr"` + Name string `xml:"Name"` + PostalRegion string `xml:"PostalRegion"` +} + +type SiteRef struct { + Ref string `xml:"ref,attr"` +} diff --git a/routes/naolib_routes.go b/routes/naolib_routes.go index 7eba9af..678dbd5 100644 --- a/routes/naolib_routes.go +++ b/routes/naolib_routes.go @@ -1,16 +1,21 @@ package routes import ( + "database/sql" + "github.com/gofiber/fiber/v2" "github.com/plugimt/transat-backend/handlers" "github.com/plugimt/transat-backend/services" ) -func SetupNaolibRoutes(router fiber.Router, naolibService *services.NaolibService) { +func SetupNaolibRoutes(router fiber.Router, naolibService *services.NaolibService, db *sql.DB) { // Groupe de routes pour Naolib naolib := router.Group("/naolib") - handler := handlers.NewNaolibHandler(naolibService) + handler := handlers.NewNaolibHandler(naolibService, db) // Route pour obtenir les prochains départs naolib.Get("/departures/chantrerie", handler.GetNextDeparturesChantrerie) + + // TODO: protéger cette route ! + naolib.Post("/import-netex-data", handler.ImportNetexData) } diff --git a/services/netex/decode.go b/services/netex/decode.go new file mode 100644 index 0000000..3e4451f --- /dev/null +++ b/services/netex/decode.go @@ -0,0 +1,118 @@ +package netex + +import ( + "archive/zip" + "bytes" + "encoding/xml" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/google/uuid" + "github.com/plugimt/transat-backend/models" + "github.com/plugimt/transat-backend/utils" +) + +func DownloadAndExtractIfNeeded(url string) (string, error) { + // download the file + resp, err := http.Get(url) + if err != nil { + return "", err + } + + // save the file to the local filesystem to /tmp/ + fileName := fmt.Sprintf("/tmp/%s", uuid.New().String()) + file, err := os.Create(fileName) + if err != nil { + return "", err + } + defer file.Close() + io.Copy(file, resp.Body) + defer resp.Body.Close() + defer os.Remove(fileName) + + // check the file's first 4 bytes to see if it's a ZIP file + zipHeader := []byte{0x50, 0x4B, 0x03, 0x04} + zipHeaderBytes := make([]byte, 4) + _, err = file.ReadAt(zipHeaderBytes, 0) + if err != nil { + return "", err + } + if bytes.Equal(zipHeaderBytes, zipHeader) { + utils.LogMessage(utils.LevelInfo, "💥 File is a ZIP file") + dst := fmt.Sprintf("/tmp/%s", uuid.New().String()) + archive, err := zip.OpenReader(fileName) + if err != nil { + panic(err) + } + defer archive.Close() + + if len(archive.File) == 0 || len(archive.File) > 1 { + return "", fmt.Errorf("invalid file") + } + + zipFile := archive.File[0] + + if !strings.HasSuffix(zipFile.Name, ".xml") { + return "", fmt.Errorf("invalid file") + } + + filePath := filepath.Join(dst, zipFile.Name) + fmt.Println("unzipping file ", filePath) + + if !strings.HasPrefix(filePath, filepath.Clean(dst)+string(os.PathSeparator)) { + fmt.Println("invalid file path") + return "", fmt.Errorf("invalid file path") + } + if zipFile.FileInfo().IsDir() { + return "", fmt.Errorf("file is a directory") + } + + if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { + panic(err) + } + + dstFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, zipFile.Mode()) + if err != nil { + panic(err) + } + + fileInArchive, err := zipFile.Open() + if err != nil { + panic(err) + } + + if _, err := io.Copy(dstFile, fileInArchive); err != nil { + panic(err) + } + + dstFile.Close() + fileInArchive.Close() + + // delete the zip file + os.Remove(fileName) + + fileName = filePath + + } + + return fileName, nil +} + +func DecodeNetexData(file string) (*models.PublicationDelivery, error) { + data, err := os.ReadFile(file) + if err != nil { + return nil, err + } + + var netexData models.PublicationDelivery + err = xml.Unmarshal(data, &netexData) + if err != nil { + return nil, err + } + + return &netexData, nil +} From fe8eec3a4125eebb6df717df89b3469ed4d5914c Mon Sep 17 00:00:00 2001 From: Lucie Delestre Date: Fri, 23 May 2025 21:20:25 +0200 Subject: [PATCH 03/12] =?UTF-8?q?Ajout=20g=C3=A9n=C3=A9ration=20de=20StopM?= =?UTF-8?q?onitoringRequest?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- services/netex/generateXML.go | 42 +++++++++++++++++++ .../StopMonitoringRequest.xml.template | 13 ++++++ 2 files changed, 55 insertions(+) create mode 100644 services/netex/generateXML.go create mode 100644 services/netex/template/StopMonitoringRequest.xml.template diff --git a/services/netex/generateXML.go b/services/netex/generateXML.go new file mode 100644 index 0000000..2cb7d11 --- /dev/null +++ b/services/netex/generateXML.go @@ -0,0 +1,42 @@ +package netex + +import ( + "bytes" + "embed" + "io" + "text/template" +) + +//go:embed template/*.xml.template +var templatesFS embed.FS + +func GenerateStopMonitoringRequest(stops []string) (string, error) { + templ, err := templatesFS.Open("template/StopMonitoringRequest.xml.template") + if err != nil { + return "", err + } + + content, err := io.ReadAll(templ) + if err != nil { + return "", err + } + + templateData := struct { + Stops []string + }{ + Stops: stops, + } + + tmpl, err := template.New("StopMonitoringRequest.xml.template").Parse(string(content)) + if err != nil { + return "", err + } + + var buf bytes.Buffer + err = tmpl.Execute(&buf, templateData) + if err != nil { + return "", err + } + + return buf.String(), nil +} diff --git a/services/netex/template/StopMonitoringRequest.xml.template b/services/netex/template/StopMonitoringRequest.xml.template new file mode 100644 index 0000000..d559359 --- /dev/null +++ b/services/netex/template/StopMonitoringRequest.xml.template @@ -0,0 +1,13 @@ + + + + opendata + {{ range .Stops }} + + {{ . }} + + {{ end }} + + \ No newline at end of file From 2ce01908cc0acff9c690bae77acf9d528f73bb4f Mon Sep 17 00:00:00 2001 From: Lucie Delestre Date: Fri, 23 May 2025 21:35:56 +0200 Subject: [PATCH 04/12] =?UTF-8?q?Ajout=20recherche=20arr=C3=AAt=20et=20g?= =?UTF-8?q?=C3=A9n=C3=A9rateur=20de=20requ=C3=AAte=20pour=20tester?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handlers/naolib.go | 66 +++++++++---------- routes/naolib_routes.go | 4 ++ services/netex/consts.go | 7 ++ services/netex/decode.go | 59 +++++++++++++++++ services/netex/generateXML.go | 6 +- .../StopMonitoringRequest.xml.template | 2 +- 6 files changed, 108 insertions(+), 36 deletions(-) create mode 100644 services/netex/consts.go diff --git a/handlers/naolib.go b/handlers/naolib.go index e3dd45c..65dcace 100644 --- a/handlers/naolib.go +++ b/handlers/naolib.go @@ -5,6 +5,7 @@ import ( "os" "github.com/gofiber/fiber/v2" + "github.com/plugimt/transat-backend/models" "github.com/plugimt/transat-backend/services" "github.com/plugimt/transat-backend/services/netex" "github.com/plugimt/transat-backend/utils" @@ -68,61 +69,60 @@ func (h *NaolibHandler) ImportNetexData(c *fiber.Ctx) error { return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) } - stopPlaces := netexData.DataObjects.GeneralFrame.Members.StopPlaces - quays := netexData.DataObjects.GeneralFrame.Members.Quays - - // on crée une transaction et on supprime toutes les données existantes, avant de les insérer - tx, err := h.db.Begin() + err = netex.SaveNetexToDatabase(netexData, h.db) if err != nil { return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) } - defer tx.Rollback() - _, err = tx.Exec("DELETE FROM NETEX_StopPlace") - if err != nil { - return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) - } + return c.JSON(map[string]any{ + "success": true, + }) +} - _, err = tx.Exec("DELETE FROM NETEX_Quay") - if err != nil { - return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) +func (h *NaolibHandler) SearchStopPlace(c *fiber.Ctx) error { + query := c.Query("query") + if query == "" { + return c.Status(fiber.StatusBadRequest).SendString("Query is required") } - _, err = tx.Exec("DELETE FROM NETEX_StopPlace_QuayRef") + rows, err := h.db.Query("SELECT id, name FROM NETEX_StopPlace WHERE name ILIKE $1", "%"+query+"%") if err != nil { return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) } + defer rows.Close() - for _, stopPlace := range stopPlaces { - _, err := tx.Exec("INSERT INTO NETEX_StopPlace (id, modification, name, longitude, latitude, transport_mode, other_transport_modes, stop_place_type, weighting) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", stopPlace.ID, stopPlace.Modification, stopPlace.Name, stopPlace.Centroid.Location.Longitude, stopPlace.Centroid.Location.Latitude, stopPlace.TransportMode, stopPlace.OtherTransportModes, stopPlace.StopPlaceType, stopPlace.Weighting) + var stopPlaces []models.StopPlace + for rows.Next() { + var stopPlace models.StopPlace + err = rows.Scan(&stopPlace.ID, &stopPlace.Name) if err != nil { return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) } + stopPlaces = append(stopPlaces, stopPlace) } - for _, quay := range quays { - _, err := tx.Exec("INSERT INTO NETEX_Quay (id, name, longitude, latitude, site_ref_stopplace_id, transport_mode) VALUES ($1, $2, $3, $4, $5, $6)", quay.ID, quay.Name, quay.Centroid.Location.Longitude, quay.Centroid.Location.Latitude, quay.SiteRef.Ref, quay.TransportMode) - if err != nil { - return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) - } + type StopPlaceResponse struct { + ID string `json:"id"` + Name string `json:"name"` } - // now, populate the NETEX_StopPlace_QuayRef link table - for _, stopPlace := range stopPlaces { - for _, quayRef := range stopPlace.QuayRefs { - _, err := tx.Exec("INSERT INTO NETEX_StopPlace_QuayRef (stop_place_id, quay_id, quay_ref_version) VALUES ($1, $2, $3)", stopPlace.ID, quayRef.Ref, quayRef.Version) - if err != nil { - return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) - } + stopPlacesArray := make([]StopPlaceResponse, len(stopPlaces)) + for i, stopPlace := range stopPlaces { + stopPlacesArray[i] = StopPlaceResponse{ + ID: stopPlace.ID, + Name: stopPlace.Name, } } - err = tx.Commit() + return c.JSON(stopPlacesArray) +} + +func (h *NaolibHandler) GenerateNetexRequest(c *fiber.Ctx) error { + stops := []string{"CTRE2", "CTRE4"} + request, err := netex.GenerateStopMonitoringRequest(stops) if err != nil { - return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + return c.Status(fiber.StatusInternalServerError).SendString(err.Error() + "\n") } - return c.JSON(map[string]interface{}{ - "success": true, - }) + return c.SendString(request + "\n") } diff --git a/routes/naolib_routes.go b/routes/naolib_routes.go index 678dbd5..aac49a2 100644 --- a/routes/naolib_routes.go +++ b/routes/naolib_routes.go @@ -18,4 +18,8 @@ func SetupNaolibRoutes(router fiber.Router, naolibService *services.NaolibServic // TODO: protéger cette route ! naolib.Post("/import-netex-data", handler.ImportNetexData) + + naolib.Get("/search", handler.SearchStopPlace) + + naolib.Get("/generate-request", handler.GenerateNetexRequest) } diff --git a/services/netex/consts.go b/services/netex/consts.go new file mode 100644 index 0000000..98d2be4 --- /dev/null +++ b/services/netex/consts.go @@ -0,0 +1,7 @@ +package netex + +const ( + RequestorRef = "opendata" + datasetId = "NAOLIBORG" + API = "https://api.okina.fr/gateway/sem/realtime/anshar/services" +) diff --git a/services/netex/decode.go b/services/netex/decode.go index 3e4451f..d4e34d5 100644 --- a/services/netex/decode.go +++ b/services/netex/decode.go @@ -3,6 +3,7 @@ package netex import ( "archive/zip" "bytes" + "database/sql" "encoding/xml" "fmt" "io" @@ -116,3 +117,61 @@ func DecodeNetexData(file string) (*models.PublicationDelivery, error) { return &netexData, nil } + +func SaveNetexToDatabase(netexData *models.PublicationDelivery, db *sql.DB) error { + stopPlaces := netexData.DataObjects.GeneralFrame.Members.StopPlaces + quays := netexData.DataObjects.GeneralFrame.Members.Quays + + // on crée une transaction et on supprime toutes les données existantes, avant de les insérer + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + _, err = tx.Exec("DELETE FROM NETEX_StopPlace") + if err != nil { + return err + } + + _, err = tx.Exec("DELETE FROM NETEX_Quay") + if err != nil { + return err + } + + _, err = tx.Exec("DELETE FROM NETEX_StopPlace_QuayRef") + if err != nil { + return err + } + + for _, stopPlace := range stopPlaces { + _, err := tx.Exec("INSERT INTO NETEX_StopPlace (id, modification, name, longitude, latitude, transport_mode, other_transport_modes, stop_place_type, weighting) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", stopPlace.ID, stopPlace.Modification, stopPlace.Name, stopPlace.Centroid.Location.Longitude, stopPlace.Centroid.Location.Latitude, stopPlace.TransportMode, stopPlace.OtherTransportModes, stopPlace.StopPlaceType, stopPlace.Weighting) + if err != nil { + return err + } + } + + for _, quay := range quays { + _, err := tx.Exec("INSERT INTO NETEX_Quay (id, name, longitude, latitude, site_ref_stopplace_id, transport_mode) VALUES ($1, $2, $3, $4, $5, $6)", quay.ID, quay.Name, quay.Centroid.Location.Longitude, quay.Centroid.Location.Latitude, quay.SiteRef.Ref, quay.TransportMode) + if err != nil { + return err + } + } + + // now, populate the NETEX_StopPlace_QuayRef link table + for _, stopPlace := range stopPlaces { + for _, quayRef := range stopPlace.QuayRefs { + _, err := tx.Exec("INSERT INTO NETEX_StopPlace_QuayRef (stop_place_id, quay_id, quay_ref_version) VALUES ($1, $2, $3)", stopPlace.ID, quayRef.Ref, quayRef.Version) + if err != nil { + return err + } + } + } + + err = tx.Commit() + if err != nil { + return err + } + + return nil +} diff --git a/services/netex/generateXML.go b/services/netex/generateXML.go index 2cb7d11..76e8215 100644 --- a/services/netex/generateXML.go +++ b/services/netex/generateXML.go @@ -22,9 +22,11 @@ func GenerateStopMonitoringRequest(stops []string) (string, error) { } templateData := struct { - Stops []string + RequestorRef string + Stops []string }{ - Stops: stops, + RequestorRef: RequestorRef, + Stops: stops, } tmpl, err := template.New("StopMonitoringRequest.xml.template").Parse(string(content)) diff --git a/services/netex/template/StopMonitoringRequest.xml.template b/services/netex/template/StopMonitoringRequest.xml.template index d559359..2b75517 100644 --- a/services/netex/template/StopMonitoringRequest.xml.template +++ b/services/netex/template/StopMonitoringRequest.xml.template @@ -3,7 +3,7 @@ xmlns:ns3="http://www.ifopt.org.uk/ifopt" xmlns:ns4="http://da-tex2.eu/schema/2_0RC1/2_0" version="2.0"> - opendata + {{ .RequestorRef }} {{ range .Stops }} {{ . }} From 19c4de8e3089ca6490cf4dabacc8733c016fc8db Mon Sep 17 00:00:00 2001 From: Lucie Delestre Date: Fri, 23 May 2025 22:47:22 +0200 Subject: [PATCH 05/12] =?UTF-8?q?Requ=C3=AAte=20prochains=20d=C3=A9part,?= =?UTF-8?q?=20par=20ligne/direction?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handlers/naolib.go | 88 +++++++++++++++++++++++++++++++++++++++ models/siri.go | 60 ++++++++++++++++++++++++++ routes/naolib_routes.go | 2 + services/netex/consts.go | 4 +- services/netex/request.go | 77 ++++++++++++++++++++++++++++++++++ 5 files changed, 230 insertions(+), 1 deletion(-) create mode 100644 models/siri.go create mode 100644 services/netex/request.go diff --git a/handlers/naolib.go b/handlers/naolib.go index 65dcace..4c1799a 100644 --- a/handlers/naolib.go +++ b/handlers/naolib.go @@ -3,6 +3,7 @@ package handlers import ( "database/sql" "os" + "strings" "github.com/gofiber/fiber/v2" "github.com/plugimt/transat-backend/models" @@ -126,3 +127,90 @@ func (h *NaolibHandler) GenerateNetexRequest(c *fiber.Ctx) error { return c.SendString(request + "\n") } + +func (h *NaolibHandler) GetDepartures(c *fiber.Ctx) error { + stopPlaceId := c.Query("stopPlaceId") + if stopPlaceId == "" { + return c.Status(fiber.StatusBadRequest).SendString("Stop place ID is required") + } + + // get the quays from the stop place id + rows, err := h.db.Query("SELECT id FROM NETEX_Quay WHERE site_ref_stopplace_id = $1", stopPlaceId) + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + defer rows.Close() + + var quays []string + for rows.Next() { + var quay string + err = rows.Scan(&quay) + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + quays = append(quays, quay) + } + + siriResponse, err := netex.CallStopMonitoringRequest(quays) + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + + departures := siriResponse.ServiceDelivery.StopMonitoringDelivery.MonitoredStopVisits + + type DepartureDirection struct { + Direction string `json:"direction"` + Departures []models.MonitoredStopVisit `json:"departures"` + } + + type Departures struct { + DepartureDirectionAller DepartureDirection `json:"aller"` + DepartureDirectionRetour DepartureDirection `json:"retour"` + } + + departuresMap := make(map[string]Departures) + + for _, departure := range departures { + lineRef := departure.MonitoredVehicleJourney.LineRef + + lineDepartures, ok := departuresMap[lineRef] + if !ok { + lineDepartures = Departures{ + DepartureDirectionAller: DepartureDirection{ + Direction: "", + Departures: []models.MonitoredStopVisit{}, + }, + DepartureDirectionRetour: DepartureDirection{ + Direction: "", + Departures: []models.MonitoredStopVisit{}, + }, + } + } + + if departure.MonitoredVehicleJourney.DirectionName == "A" { + if lineDepartures.DepartureDirectionAller.Direction == "" { + lineDepartures.DepartureDirectionAller.Direction = departure.MonitoredVehicleJourney.DestinationName + } else { + if !strings.Contains(lineDepartures.DepartureDirectionAller.Direction, departure.MonitoredVehicleJourney.DestinationName) { + lineDepartures.DepartureDirectionAller.Direction += " / " + departure.MonitoredVehicleJourney.DestinationName + } + } + lineDepartures.DepartureDirectionAller.Departures = append(lineDepartures.DepartureDirectionAller.Departures, departure) + } else { + // si la destination n'est pas B (valeur par défaut), ça signifie qu'on l'a déjà changé. on doit aller vérifier si c'est la même destination, sinon on rajoute la destination avec "/ " + if lineDepartures.DepartureDirectionRetour.Direction == "" { + lineDepartures.DepartureDirectionRetour.Direction = departure.MonitoredVehicleJourney.DestinationName + } else { + if !strings.Contains(lineDepartures.DepartureDirectionRetour.Direction, departure.MonitoredVehicleJourney.DestinationName) { + lineDepartures.DepartureDirectionRetour.Direction += " / " + departure.MonitoredVehicleJourney.DestinationName + } + } + + lineDepartures.DepartureDirectionRetour.Departures = append(lineDepartures.DepartureDirectionRetour.Departures, departure) + } + + departuresMap[lineRef] = lineDepartures + } + + return c.JSON(departuresMap) +} diff --git a/models/siri.go b/models/siri.go new file mode 100644 index 0000000..5f1d769 --- /dev/null +++ b/models/siri.go @@ -0,0 +1,60 @@ +package models + +import ( + "encoding/xml" + "time" +) + +type SIRI struct { + XMLName xml.Name `xml:"Siri"` + ServiceDelivery ServiceDelivery `xml:"ServiceDelivery"` +} + +type ServiceDelivery struct { + ResponseTimestamp time.Time `xml:"ResponseTimestamp"` + ProducerRef string `xml:"ProducerRef"` + RequestMessageRef string `xml:"RequestMessageRef"` + Status bool `xml:"Status"` + MoreData bool `xml:"MoreData"` + StopMonitoringDelivery StopMonitoringDelivery `xml:"StopMonitoringDelivery"` +} + +type StopMonitoringDelivery struct { + ResponseTimestamp time.Time `xml:"ResponseTimestamp"` + MonitoredStopVisits []MonitoredStopVisit `xml:"MonitoredStopVisit"` +} + +type MonitoredStopVisit struct { + RecordedAtTime time.Time `xml:"RecordedAtTime"` + ItemIdentifier string `xml:"ItemIdentifier"` + MonitoringRef string `xml:"MonitoringRef"` + MonitoredVehicleJourney MonitoredVehicleJourney `xml:"MonitoredVehicleJourney"` +} + +type MonitoredVehicleJourney struct { + LineRef string `xml:"LineRef"` + FramedVehicleJourneyRef FramedVehicleJourneyRef `xml:"FramedVehicleJourneyRef"` + VehicleMode string `xml:"VehicleMode"` + PublishedLineName string `xml:"PublishedLineName"` + DirectionName string `xml:"DirectionName"` + DestinationRef string `xml:"DestinationRef"` + DestinationName string `xml:"DestinationName"` + FirstOrLastJourney string `xml:"FirstOrLastJourney"` + Monitored bool `xml:"Monitored"` + MonitoredCall MonitoredCall `xml:"MonitoredCall"` +} + +type FramedVehicleJourneyRef struct { + DataFrameRef string `xml:"DataFrameRef"` + DatedVehicleJourneyRef string `xml:"DatedVehicleJourneyRef"` +} + +type MonitoredCall struct { + StopPointRef string `xml:"StopPointRef"` + Order int `xml:"Order"` + AimedArrivalTime time.Time `xml:"AimedArrivalTime"` + ExpectedArrivalTime time.Time `xml:"ExpectedArrivalTime"` + ArrivalStatus string `xml:"ArrivalStatus"` + AimedDepartureTime time.Time `xml:"AimedDepartureTime"` + ExpectedDepartureTime time.Time `xml:"ExpectedDepartureTime"` +} diff --git a/routes/naolib_routes.go b/routes/naolib_routes.go index aac49a2..7cfd731 100644 --- a/routes/naolib_routes.go +++ b/routes/naolib_routes.go @@ -22,4 +22,6 @@ func SetupNaolibRoutes(router fiber.Router, naolibService *services.NaolibServic naolib.Get("/search", handler.SearchStopPlace) naolib.Get("/generate-request", handler.GenerateNetexRequest) + + naolib.Get("/get-departures", handler.GetDepartures) } diff --git a/services/netex/consts.go b/services/netex/consts.go index 98d2be4..c9e200d 100644 --- a/services/netex/consts.go +++ b/services/netex/consts.go @@ -3,5 +3,7 @@ package netex const ( RequestorRef = "opendata" datasetId = "NAOLIBORG" - API = "https://api.okina.fr/gateway/sem/realtime/anshar/services" + APIScheme = "https" + APIHost = "api.okina.fr" + APIPath = "/gateway/sem/realtime/anshar/services" ) diff --git a/services/netex/request.go b/services/netex/request.go new file mode 100644 index 0000000..9e42bc4 --- /dev/null +++ b/services/netex/request.go @@ -0,0 +1,77 @@ +package netex + +import ( + "bytes" + "encoding/xml" + "fmt" + "io" + "net/http" + "net/url" + "sync" + "time" + + "github.com/plugimt/transat-backend/models" +) + +var httpClient = &http.Client{ + Timeout: 10 * time.Second, +} + +var mu sync.Mutex +var lastRequestTime time.Time + +func CallStopMonitoringRequest(stops []string) (*models.SIRI, error) { + mu.Lock() + defer mu.Unlock() + + if time.Since(lastRequestTime) < 1*time.Second { + time.Sleep(1 * time.Second) + } + + content, err := GenerateStopMonitoringRequest(stops) + if err != nil { + return nil, err + } + + url := url.URL{ + Scheme: APIScheme, + Host: APIHost, + Path: APIPath, + } + + request := http.Request{ + Method: "POST", + URL: &url, + Body: io.NopCloser(bytes.NewBufferString(content)), + Header: http.Header{ + "Content-Type": []string{"application/xml"}, + "datasetId": []string{datasetId}, + }, + } + + resp, err := httpClient.Do(&request) + if err != nil { + return nil, err + } + + lastRequestTime = time.Now() + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("status code: %d", resp.StatusCode) + } + + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + var siri models.SIRI + err = xml.Unmarshal(body, &siri) + if err != nil { + return nil, err + } + + return &siri, nil +} From c7590dfa90e5b345f506b06c20311f1fcc48c76d Mon Sep 17 00:00:00 2001 From: Lucie Delestre Date: Fri, 23 May 2025 23:03:25 +0200 Subject: [PATCH 06/12] =?UTF-8?q?Passage=20=C3=A0=20la=20nouvelle=20API?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handlers/naolib.go | 98 ++++------------------------------------------ main.go | 2 +- models/naolib.go | 11 ++++++ services/naolib.go | 94 +++++++++++++++++++++++++++++++------------- 4 files changed, 87 insertions(+), 118 deletions(-) diff --git a/handlers/naolib.go b/handlers/naolib.go index 4c1799a..ed4dd09 100644 --- a/handlers/naolib.go +++ b/handlers/naolib.go @@ -3,13 +3,15 @@ package handlers import ( "database/sql" "os" - "strings" "github.com/gofiber/fiber/v2" "github.com/plugimt/transat-backend/models" "github.com/plugimt/transat-backend/services" "github.com/plugimt/transat-backend/services/netex" - "github.com/plugimt/transat-backend/utils" +) + +const ( + ChantrerieStopPlaceId = "FR_NAOLIB:StopPlace:244" ) type NaolibHandler struct { @@ -25,23 +27,12 @@ func NewNaolibHandler(service *services.NaolibService, db *sql.DB) *NaolibHandle } func (h *NaolibHandler) GetNextDeparturesChantrerie(c *fiber.Ctx) error { - departuresC6, err := h.service.GetNextDepartures("CTRE2") - if err != nil { - utils.LogMessage(utils.LevelError, "Error fetching departures: "+err.Error()) - return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{ - "error": "Failed to fetch departure information", - }) - } - departures75, err := h.service.GetNextDepartures("CTRE4") + departures, err := h.service.GetDepartures(ChantrerieStopPlaceId) if err != nil { - utils.LogMessage(utils.LevelError, "Error fetching departures: "+err.Error()) + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) } - departures := append(departuresC6, departures75...) - return c.JSON(fiber.Map{ - "success": true, - "data": departures, - }) + return c.JSON(departures) } func (h *NaolibHandler) ImportNetexData(c *fiber.Ctx) error { @@ -134,83 +125,10 @@ func (h *NaolibHandler) GetDepartures(c *fiber.Ctx) error { return c.Status(fiber.StatusBadRequest).SendString("Stop place ID is required") } - // get the quays from the stop place id - rows, err := h.db.Query("SELECT id FROM NETEX_Quay WHERE site_ref_stopplace_id = $1", stopPlaceId) + departuresMap, err := h.service.GetDepartures(stopPlaceId) if err != nil { return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) } - defer rows.Close() - - var quays []string - for rows.Next() { - var quay string - err = rows.Scan(&quay) - if err != nil { - return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) - } - quays = append(quays, quay) - } - - siriResponse, err := netex.CallStopMonitoringRequest(quays) - if err != nil { - return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) - } - - departures := siriResponse.ServiceDelivery.StopMonitoringDelivery.MonitoredStopVisits - - type DepartureDirection struct { - Direction string `json:"direction"` - Departures []models.MonitoredStopVisit `json:"departures"` - } - - type Departures struct { - DepartureDirectionAller DepartureDirection `json:"aller"` - DepartureDirectionRetour DepartureDirection `json:"retour"` - } - - departuresMap := make(map[string]Departures) - - for _, departure := range departures { - lineRef := departure.MonitoredVehicleJourney.LineRef - - lineDepartures, ok := departuresMap[lineRef] - if !ok { - lineDepartures = Departures{ - DepartureDirectionAller: DepartureDirection{ - Direction: "", - Departures: []models.MonitoredStopVisit{}, - }, - DepartureDirectionRetour: DepartureDirection{ - Direction: "", - Departures: []models.MonitoredStopVisit{}, - }, - } - } - - if departure.MonitoredVehicleJourney.DirectionName == "A" { - if lineDepartures.DepartureDirectionAller.Direction == "" { - lineDepartures.DepartureDirectionAller.Direction = departure.MonitoredVehicleJourney.DestinationName - } else { - if !strings.Contains(lineDepartures.DepartureDirectionAller.Direction, departure.MonitoredVehicleJourney.DestinationName) { - lineDepartures.DepartureDirectionAller.Direction += " / " + departure.MonitoredVehicleJourney.DestinationName - } - } - lineDepartures.DepartureDirectionAller.Departures = append(lineDepartures.DepartureDirectionAller.Departures, departure) - } else { - // si la destination n'est pas B (valeur par défaut), ça signifie qu'on l'a déjà changé. on doit aller vérifier si c'est la même destination, sinon on rajoute la destination avec "/ " - if lineDepartures.DepartureDirectionRetour.Direction == "" { - lineDepartures.DepartureDirectionRetour.Direction = departure.MonitoredVehicleJourney.DestinationName - } else { - if !strings.Contains(lineDepartures.DepartureDirectionRetour.Direction, departure.MonitoredVehicleJourney.DestinationName) { - lineDepartures.DepartureDirectionRetour.Direction += " / " + departure.MonitoredVehicleJourney.DestinationName - } - } - - lineDepartures.DepartureDirectionRetour.Departures = append(lineDepartures.DepartureDirectionRetour.Departures, departure) - } - - departuresMap[lineRef] = lineDepartures - } return c.JSON(departuresMap) } diff --git a/main.go b/main.go index ab9f30d..23fd975 100644 --- a/main.go +++ b/main.go @@ -119,7 +119,7 @@ func main() { // Initialize Handlers that need explicit instantiation (e.g., for Cron) restHandler := restaurantHandler.NewRestaurantHandler(db, translationService, notificationService) - naolibService := services.NewNaolibService(30 * time.Second) + naolibService := services.NewNaolibService(db, 30*time.Second) // Initialize Weather Service and Handler weatherService, err := services.NewWeatherService() diff --git a/models/naolib.go b/models/naolib.go index b0450fe..5ac6c51 100644 --- a/models/naolib.go +++ b/models/naolib.go @@ -22,3 +22,14 @@ type Ligne struct { type Arret struct { CodeArret string `json:"codeArret"` } + +// custom +type DepartureDirection struct { + Direction string `json:"direction"` + Departures []MonitoredStopVisit `json:"departures"` +} + +type Departures struct { + DepartureDirectionAller DepartureDirection `json:"aller"` + DepartureDirectionRetour DepartureDirection `json:"retour"` +} diff --git a/services/naolib.go b/services/naolib.go index b30c2d7..c9f3cd2 100644 --- a/services/naolib.go +++ b/services/naolib.go @@ -1,14 +1,14 @@ package services import ( - "encoding/json" - "fmt" + "database/sql" "net/http" + "strings" "sync" "time" "github.com/plugimt/transat-backend/models" - "github.com/plugimt/transat-backend/utils" + "github.com/plugimt/transat-backend/services/netex" ) var httpClient = &http.Client{ @@ -20,46 +20,86 @@ type NaolibService struct { cache map[string][]models.Departure cacheTime time.Duration lastUpdate map[string]time.Time + db *sql.DB } -func NewNaolibService(refreshTime time.Duration) *NaolibService { +func NewNaolibService(db *sql.DB, refreshTime time.Duration) *NaolibService { return &NaolibService{ + db: db, cache: make(map[string][]models.Departure), cacheTime: refreshTime, lastUpdate: make(map[string]time.Time), } } -func (s *NaolibService) GetNextDepartures(stopID string) ([]models.Departure, error) { - s.mu.Lock() - if lastUpdateTime, updateExists := s.lastUpdate[stopID]; updateExists { - if departures, ok := s.cache[stopID]; ok && time.Since(lastUpdateTime) < s.cacheTime { - s.mu.Unlock() - utils.LogMessage(utils.LevelInfo, fmt.Sprintf("Using cached departures for %s", stopID)) - return departures, nil - } +func (s *NaolibService) GetDepartures(stopPlaceId string) (map[string]models.Departures, error) { + rows, err := s.db.Query("SELECT id FROM NETEX_Quay WHERE site_ref_stopplace_id = $1", stopPlaceId) + if err != nil { + return nil, err } + defer rows.Close() - url := fmt.Sprintf("https://open.tan.fr/ewp/tempsattentelieu.json/%s/2", stopID) + var quays []string + for rows.Next() { + var quay string + err = rows.Scan(&quay) + if err != nil { + return nil, err + } + quays = append(quays, quay) + } - resp, err := httpClient.Get(url) + siriResponse, err := netex.CallStopMonitoringRequest(quays) if err != nil { - return nil, fmt.Errorf("erreur lors de la requête HTTP: %v", err) + return nil, err } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("erreur HTTP: code %d", resp.StatusCode) - } + departures := siriResponse.ServiceDelivery.StopMonitoringDelivery.MonitoredStopVisits - var departuresData []models.Departure - if err := json.NewDecoder(resp.Body).Decode(&departuresData); err != nil { - return nil, fmt.Errorf("erreur lors du décodage JSON: %v", err) - } + departuresMap := make(map[string]models.Departures) + + for _, departure := range departures { + lineRef := departure.MonitoredVehicleJourney.LineRef - s.cache[stopID] = departuresData - s.lastUpdate[stopID] = time.Now() - s.mu.Unlock() + lineDepartures, ok := departuresMap[lineRef] + if !ok { + lineDepartures = models.Departures{ + DepartureDirectionAller: models.DepartureDirection{ + Direction: "", + Departures: []models.MonitoredStopVisit{}, + }, + DepartureDirectionRetour: models.DepartureDirection{ + Direction: "", + Departures: []models.MonitoredStopVisit{}, + }, + } + } + + if departure.MonitoredVehicleJourney.DirectionName == "A" { + // si la destination n'est pas "" (valeur par défaut), ça signifie qu'on l'a déjà changé. on doit aller vérifier + // si c'est la même destination, sinon on rajoute la destination avec "/ " + if lineDepartures.DepartureDirectionAller.Direction == "" { + lineDepartures.DepartureDirectionAller.Direction = departure.MonitoredVehicleJourney.DestinationName + } else { + if !strings.Contains(lineDepartures.DepartureDirectionAller.Direction, departure.MonitoredVehicleJourney.DestinationName) { + lineDepartures.DepartureDirectionAller.Direction += " / " + departure.MonitoredVehicleJourney.DestinationName + } + } + lineDepartures.DepartureDirectionAller.Departures = append(lineDepartures.DepartureDirectionAller.Departures, departure) + } else { + if lineDepartures.DepartureDirectionRetour.Direction == "" { + lineDepartures.DepartureDirectionRetour.Direction = departure.MonitoredVehicleJourney.DestinationName + } else { + if !strings.Contains(lineDepartures.DepartureDirectionRetour.Direction, departure.MonitoredVehicleJourney.DestinationName) { + lineDepartures.DepartureDirectionRetour.Direction += " / " + departure.MonitoredVehicleJourney.DestinationName + } + } + + lineDepartures.DepartureDirectionRetour.Departures = append(lineDepartures.DepartureDirectionRetour.Departures, departure) + } + + departuresMap[lineRef] = lineDepartures + } - return departuresData, nil + return departuresMap, nil } From 954c465b966c49298b7dc1b98409656252fe7809 Mon Sep 17 00:00:00 2001 From: Lucie Delestre Date: Fri, 23 May 2025 23:04:19 +0200 Subject: [PATCH 07/12] nettoyage --- models/naolib.go | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/models/naolib.go b/models/naolib.go index 5ac6c51..8890682 100644 --- a/models/naolib.go +++ b/models/naolib.go @@ -1,29 +1,5 @@ package models -// Departure représente un départ de bus -type Departure struct { - Sens int `json:"sens"` - Terminus string `json:"terminus"` - InfoTrafic bool `json:"infotrafic"` - Temps string `json:"temps"` - DernierDepart string `json:"dernierDepart"` - TempsReel string `json:"tempsReel"` - Ligne Ligne `json:"ligne"` - Arret Arret `json:"arret"` -} - -// Ligne représente les informations sur la ligne de bus -type Ligne struct { - NumLigne string `json:"numLigne"` - TypeLigne int `json:"typeLigne"` -} - -// Arret représente les informations sur l'arrêt de bus -type Arret struct { - CodeArret string `json:"codeArret"` -} - -// custom type DepartureDirection struct { Direction string `json:"direction"` Departures []MonitoredStopVisit `json:"departures"` From 2d2b84cf1b76f679a23cb605bee658f21e6854db Mon Sep 17 00:00:00 2001 From: Lucie Delestre Date: Fri, 23 May 2025 23:06:31 +0200 Subject: [PATCH 08/12] nettoyage p2 --- main.go | 2 +- services/naolib.go | 15 ++++----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/main.go b/main.go index 23fd975..1788bb6 100644 --- a/main.go +++ b/main.go @@ -119,7 +119,7 @@ func main() { // Initialize Handlers that need explicit instantiation (e.g., for Cron) restHandler := restaurantHandler.NewRestaurantHandler(db, translationService, notificationService) - naolibService := services.NewNaolibService(db, 30*time.Second) + naolibService := services.NewNaolibService(db) // Initialize Weather Service and Handler weatherService, err := services.NewWeatherService() diff --git a/services/naolib.go b/services/naolib.go index c9f3cd2..21e9b56 100644 --- a/services/naolib.go +++ b/services/naolib.go @@ -4,7 +4,6 @@ import ( "database/sql" "net/http" "strings" - "sync" "time" "github.com/plugimt/transat-backend/models" @@ -16,23 +15,17 @@ var httpClient = &http.Client{ } type NaolibService struct { - mu sync.Mutex - cache map[string][]models.Departure - cacheTime time.Duration - lastUpdate map[string]time.Time - db *sql.DB + db *sql.DB } -func NewNaolibService(db *sql.DB, refreshTime time.Duration) *NaolibService { +func NewNaolibService(db *sql.DB) *NaolibService { return &NaolibService{ - db: db, - cache: make(map[string][]models.Departure), - cacheTime: refreshTime, - lastUpdate: make(map[string]time.Time), + db: db, } } func (s *NaolibService) GetDepartures(stopPlaceId string) (map[string]models.Departures, error) { + rows, err := s.db.Query("SELECT id FROM NETEX_Quay WHERE site_ref_stopplace_id = $1", stopPlaceId) if err != nil { return nil, err From d0f036d32dcc07dc6c19c2d08fa1c223f8642d96 Mon Sep 17 00:00:00 2001 From: Lucie Delestre Date: Fri, 23 May 2025 23:21:10 +0200 Subject: [PATCH 09/12] mapper pour ne pas renvoyer du SIRI direct --- models/naolib.go | 15 +++++++++++++-- services/naolib.go | 23 +++++++++++++++++++++-- 2 files changed, 34 insertions(+), 4 deletions(-) diff --git a/models/naolib.go b/models/naolib.go index 8890682..6e3f294 100644 --- a/models/naolib.go +++ b/models/naolib.go @@ -1,11 +1,22 @@ package models +import "time" + type DepartureDirection struct { - Direction string `json:"direction"` - Departures []MonitoredStopVisit `json:"departures"` + Direction string `json:"direction"` + Departures []Departure `json:"departures"` } type Departures struct { DepartureDirectionAller DepartureDirection `json:"aller"` DepartureDirectionRetour DepartureDirection `json:"retour"` } + +type Departure struct { + LineRef string `json:"lineRef"` + Direction string `json:"direction"` + DestinationName string `json:"destinationName"` + DepartureTime time.Time `json:"departureTime"` + ArrivalTime time.Time `json:"arrivalTime"` + VehicleMode string `json:"vehicleMode"` +} diff --git a/services/naolib.go b/services/naolib.go index 21e9b56..1447421 100644 --- a/services/naolib.go +++ b/services/naolib.go @@ -59,11 +59,11 @@ func (s *NaolibService) GetDepartures(stopPlaceId string) (map[string]models.Dep lineDepartures = models.Departures{ DepartureDirectionAller: models.DepartureDirection{ Direction: "", - Departures: []models.MonitoredStopVisit{}, + Departures: []models.Departure{}, }, DepartureDirectionRetour: models.DepartureDirection{ Direction: "", - Departures: []models.MonitoredStopVisit{}, + Departures: []models.Departure{}, }, } } @@ -78,6 +78,16 @@ func (s *NaolibService) GetDepartures(stopPlaceId string) (map[string]models.Dep lineDepartures.DepartureDirectionAller.Direction += " / " + departure.MonitoredVehicleJourney.DestinationName } } + + departure := models.Departure{ + LineRef: lineRef, + Direction: lineDepartures.DepartureDirectionAller.Direction, + DestinationName: departure.MonitoredVehicleJourney.DestinationName, + DepartureTime: departure.MonitoredVehicleJourney.MonitoredCall.ExpectedDepartureTime, + ArrivalTime: departure.MonitoredVehicleJourney.MonitoredCall.ExpectedArrivalTime, + VehicleMode: departure.MonitoredVehicleJourney.VehicleMode, + } + lineDepartures.DepartureDirectionAller.Departures = append(lineDepartures.DepartureDirectionAller.Departures, departure) } else { if lineDepartures.DepartureDirectionRetour.Direction == "" { @@ -88,6 +98,15 @@ func (s *NaolibService) GetDepartures(stopPlaceId string) (map[string]models.Dep } } + departure := models.Departure{ + LineRef: lineRef, + Direction: lineDepartures.DepartureDirectionRetour.Direction, + DestinationName: departure.MonitoredVehicleJourney.DestinationName, + DepartureTime: departure.MonitoredVehicleJourney.MonitoredCall.ExpectedDepartureTime, + ArrivalTime: departure.MonitoredVehicleJourney.MonitoredCall.ExpectedArrivalTime, + VehicleMode: departure.MonitoredVehicleJourney.VehicleMode, + } + lineDepartures.DepartureDirectionRetour.Departures = append(lineDepartures.DepartureDirectionRetour.Departures, departure) } From f837f1b9bb7bd7ab63675bc256f4a642e30d7272 Mon Sep 17 00:00:00 2001 From: Lucie Delestre Date: Fri, 23 May 2025 23:51:46 +0200 Subject: [PATCH 10/12] Ajout import offer --- db/migrations/00009_add_netex_data.sql | 14 +++ handlers/naolib.go | 45 ++++++- models/netex.go | 125 ++++++++++++++++++- routes/naolib_routes.go | 5 +- services/netex/decode.go | 158 ++++++++++++++++++++++++- 5 files changed, 339 insertions(+), 8 deletions(-) diff --git a/db/migrations/00009_add_netex_data.sql b/db/migrations/00009_add_netex_data.sql index 018fc9c..54d90d9 100644 --- a/db/migrations/00009_add_netex_data.sql +++ b/db/migrations/00009_add_netex_data.sql @@ -39,6 +39,19 @@ CREATE TABLE NETEX_StopPlace_QuayRef ( UNIQUE (stop_place_id, quay_id) ); +CREATE TABLE NETEX_Line ( + id TEXT PRIMARY KEY, -- ex: "NAOLIBORG:Line:3B:LOC" + version TEXT, + name TEXT, + short_name TEXT, + transport_mode TEXT, + public_code TEXT, + private_code TEXT, + colour TEXT, + text_colour TEXT, + route_sort_order INTEGER, + UNIQUE (id) +); -- +goose StatementEnd @@ -47,4 +60,5 @@ CREATE TABLE NETEX_StopPlace_QuayRef ( DROP TABLE NETEX_StopPlace; DROP TABLE NETEX_Quay; DROP TABLE NETEX_StopPlace_QuayRef; +DROP TABLE NETEX_Line; -- +goose StatementEnd diff --git a/handlers/naolib.go b/handlers/naolib.go index ed4dd09..7eca4fa 100644 --- a/handlers/naolib.go +++ b/handlers/naolib.go @@ -2,6 +2,7 @@ package handlers import ( "database/sql" + "fmt" "os" "github.com/gofiber/fiber/v2" @@ -35,7 +36,45 @@ func (h *NaolibHandler) GetNextDeparturesChantrerie(c *fiber.Ctx) error { return c.JSON(departures) } -func (h *NaolibHandler) ImportNetexData(c *fiber.Ctx) error { +func (h *NaolibHandler) ImportNetexOffer(c *fiber.Ctx) error { + var body struct { + Url string `json:"url"` + } + + if err := c.BodyParser(&body); err != nil { + return c.Status(fiber.StatusBadRequest).SendString(err.Error()) + } + + url := body.Url + + if url == "" { + return c.Status(fiber.StatusBadRequest).SendString("URL is required") + } + + fileName, err := netex.DownloadAndExtractIfNeededOffer(url) + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + defer os.Remove(fileName) + + netexData, err := netex.DecodeNetexOfferData(fileName) + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + + fmt.Println(netexData) + + err = netex.SaveNetexOfferToDatabase(netexData, h.db) + if err != nil { + return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) + } + + return c.JSON(map[string]any{ + "success": true, + }) +} + +func (h *NaolibHandler) ImportNetexStops(c *fiber.Ctx) error { var body struct { Url string `json:"url"` } @@ -56,12 +95,12 @@ func (h *NaolibHandler) ImportNetexData(c *fiber.Ctx) error { } defer os.Remove(fileName) - netexData, err := netex.DecodeNetexData(fileName) + netexData, err := netex.DecodeNetexStopsData(fileName) if err != nil { return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) } - err = netex.SaveNetexToDatabase(netexData, h.db) + err = netex.SaveNetexStopsToDatabase(netexData, h.db) if err != nil { return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) } diff --git a/models/netex.go b/models/netex.go index c5eed4f..21d60e1 100644 --- a/models/netex.go +++ b/models/netex.go @@ -1,6 +1,9 @@ package models -import "encoding/xml" +import ( + "encoding/xml" + "os" +) type NETEXStopsFile struct { XMLName xml.Name `xml:"PublicationDelivery"` @@ -99,3 +102,123 @@ type PostalAddress struct { type SiteRef struct { Ref string `xml:"ref,attr"` } + +type NETEXCommonFile struct { + XMLName xml.Name `xml:"PublicationDelivery"` + PublicationTimestamp string `xml:"PublicationTimestamp"` + ParticipantRef string `xml:"ParticipantRef"` + DataObjects CommonDataObjects `xml:"dataObjects"` +} + +type CommonDataObjects struct { + GeneralFrame GeneralFrameOffer `xml:"GeneralFrame"` +} + +type GeneralFrameOffer struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr"` + TypeOfFrameRef TypeOfFrameRef `xml:"TypeOfFrameRef"` + Members struct { + Network Network `xml:"Network"` + Lines []Line `xml:"Line"` + Operators []Operator `xml:"Operator"` + Authorities []Authority `xml:"Authority"` + } `xml:"members"` +} + +type Network struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr"` + Name string `xml:"Name"` + Members []LineRef `xml:"members>LineRef"` + AuthorityRef AuthorityRef `xml:"AuthorityRef"` +} + +type LineRef struct { + Ref string `xml:"ref,attr"` + Version string `xml:"version,attr,omitempty"` +} + +type AuthorityRef struct { + Ref string `xml:"ref,attr"` + Version string `xml:"version,attr,omitempty"` +} + +type Line struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr"` + KeyList []KeyValue `xml:"keyList>KeyValue"` + Name string `xml:"Name"` + ShortName string `xml:"ShortName"` + TransportMode string `xml:"TransportMode"` + PublicCode string `xml:"PublicCode"` + PrivateCode string `xml:"PrivateCode"` + OperatorRef LineRef `xml:"OperatorRef"` + Presentation Presentation `xml:"Presentation"` + AccessibilityAssessment AccessibilityAssessment `xml:"AccessibilityAssessment"` +} + +type KeyValue struct { + Key string `xml:"Key"` + Value string `xml:"Value"` +} + +type Presentation struct { + Colour string `xml:"Colour"` + TextColour string `xml:"TextColour"` +} + +type AccessibilityAssessment struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr"` + MobilityImpairedAccess string `xml:"MobilityImpairedAccess"` + Limitations Limitations `xml:"limitations"` +} + +type Limitations struct { + AccessibilityLimitation AccessibilityLimitation `xml:"AccessibilityLimitation"` +} + +type AccessibilityLimitation struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr"` + WheelchairAccess string `xml:"WheelchairAccess"` +} + +type Operator struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr"` + CompanyNumber string `xml:"CompanyNumber"` + Name string `xml:"Name"` + ContactDetails ContactDetails `xml:"ContactDetails"` + OrganisationType string `xml:"OrganisationType"` +} + +type Authority struct { + ID string `xml:"id,attr"` + Version string `xml:"version,attr"` + CompanyNumber string `xml:"CompanyNumber"` + Name string `xml:"Name"` + ContactDetails ContactDetails `xml:"ContactDetails"` + OrganisationType string `xml:"OrganisationType"` +} + +type ContactDetails struct { + Phone string `xml:"Phone"` + Url string `xml:"Url"` +} + +func DecodeCommonNetexData(file string) (*NETEXCommonFile, error) { + data, err := os.ReadFile(file) + if err != nil { + return nil, err + } + + var netexData NETEXCommonFile + err = xml.Unmarshal(data, &netexData) + if err != nil { + return nil, err + } + + return &netexData, nil +} diff --git a/routes/naolib_routes.go b/routes/naolib_routes.go index 7cfd731..ae6f4cb 100644 --- a/routes/naolib_routes.go +++ b/routes/naolib_routes.go @@ -16,8 +16,9 @@ func SetupNaolibRoutes(router fiber.Router, naolibService *services.NaolibServic // Route pour obtenir les prochains départs naolib.Get("/departures/chantrerie", handler.GetNextDeparturesChantrerie) - // TODO: protéger cette route ! - naolib.Post("/import-netex-data", handler.ImportNetexData) + // TODO: protéger ces routes ! + naolib.Post("/import/netex/stops", handler.ImportNetexStops) + naolib.Post("/import/netex/offer", handler.ImportNetexOffer) naolib.Get("/search", handler.SearchStopPlace) diff --git a/services/netex/decode.go b/services/netex/decode.go index d4e34d5..b97226b 100644 --- a/services/netex/decode.go +++ b/services/netex/decode.go @@ -10,6 +10,7 @@ import ( "net/http" "os" "path/filepath" + "strconv" "strings" "github.com/google/uuid" @@ -103,7 +104,103 @@ func DownloadAndExtractIfNeeded(url string) (string, error) { return fileName, nil } -func DecodeNetexData(file string) (*models.PublicationDelivery, error) { +func DownloadAndExtractIfNeededOffer(url string) (string, error) { + // download the file + resp, err := http.Get(url) + if err != nil { + return "", err + } + + // save the file to the local filesystem to /tmp/ + fileName := fmt.Sprintf("/tmp/%s", uuid.New().String()) + file, err := os.Create(fileName) + if err != nil { + return "", err + } + defer file.Close() + io.Copy(file, resp.Body) + defer resp.Body.Close() + defer os.Remove(fileName) + + // check the file's first 4 bytes to see if it's a ZIP file + zipHeader := []byte{0x50, 0x4B, 0x03, 0x04} + zipHeaderBytes := make([]byte, 4) + _, err = file.ReadAt(zipHeaderBytes, 0) + if err != nil { + return "", err + } + if !bytes.Equal(zipHeaderBytes, zipHeader) { + return "", fmt.Errorf("invalid file") + } + + utils.LogMessage(utils.LevelInfo, "💥 File is a ZIP file") + dst := fmt.Sprintf("/tmp/%s", uuid.New().String()) + archive, err := zip.OpenReader(fileName) + if err != nil { + panic(err) + } + defer archive.Close() + + if len(archive.File) == 0 { + return "", fmt.Errorf("invalid file") + } + + // this time, the ZIP contains a folder which contains a lot of XML files. We are only interested in the xxx_commun.xml file. + // we need to unzip the file and return the path to the xxx_commun.xml file. + + // find the xxx_commun.xml file + var communFileName string + for _, f := range archive.File { + filePath := filepath.Join(dst, f.Name) + if !strings.HasSuffix(filePath, "_commun.xml") { + continue + } + + fmt.Println("unzipping file ", filePath) + + if !strings.HasPrefix(filePath, filepath.Clean(dst)+string(os.PathSeparator)) { + fmt.Println("invalid file path") + return "", fmt.Errorf("invalid file path") + } + if f.FileInfo().IsDir() { + fmt.Println("creating directory...") + os.MkdirAll(filePath, os.ModePerm) + continue + } + + if err := os.MkdirAll(filepath.Dir(filePath), os.ModePerm); err != nil { + panic(err) + } + + fmt.Println("opening file ", filePath) + dstFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + panic(err) + } + + fileInArchive, err := f.Open() + if err != nil { + panic(err) + } + + fmt.Println("writing file ", filePath) + if _, err := io.Copy(dstFile, fileInArchive); err != nil { + panic(err) + } + + dstFile.Close() + fileInArchive.Close() + + communFileName = filePath + } + + // delete the zip file + os.Remove(fileName) + + return communFileName, nil +} + +func DecodeNetexStopsData(file string) (*models.PublicationDelivery, error) { data, err := os.ReadFile(file) if err != nil { return nil, err @@ -118,7 +215,22 @@ func DecodeNetexData(file string) (*models.PublicationDelivery, error) { return &netexData, nil } -func SaveNetexToDatabase(netexData *models.PublicationDelivery, db *sql.DB) error { +func DecodeNetexOfferData(fileName string) (*models.NETEXCommonFile, error) { + data, err := os.ReadFile(fileName) + if err != nil { + return nil, err + } + + var netexData models.NETEXCommonFile + err = xml.Unmarshal(data, &netexData) + if err != nil { + return nil, err + } + + return &netexData, nil +} + +func SaveNetexStopsToDatabase(netexData *models.PublicationDelivery, db *sql.DB) error { stopPlaces := netexData.DataObjects.GeneralFrame.Members.StopPlaces quays := netexData.DataObjects.GeneralFrame.Members.Quays @@ -175,3 +287,45 @@ func SaveNetexToDatabase(netexData *models.PublicationDelivery, db *sql.DB) erro return nil } + +func SaveNetexOfferToDatabase(netexData *models.NETEXCommonFile, db *sql.DB) error { + lines := netexData.DataObjects.GeneralFrame.Members.Lines + + // on crée une transaction et on supprime toutes les données existantes, avant de les insérer + tx, err := db.Begin() + if err != nil { + return err + } + defer tx.Rollback() + + _, err = tx.Exec("DELETE FROM NETEX_Line") + if err != nil { + return err + } + + for _, line := range lines { + routeSortOrder := 0 + if line.KeyList != nil { + for _, keyValue := range line.KeyList { + if keyValue.Key == "route_sort_order" { + routeSortOrder, err = strconv.Atoi(keyValue.Value) + if err != nil { + return err + } + } + } + } + + _, err := tx.Exec("INSERT INTO NETEX_Line (id, version, name, short_name, transport_mode, public_code, private_code, colour, text_colour, route_sort_order) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)", line.ID, line.Version, line.Name, line.ShortName, line.TransportMode, line.PublicCode, line.PrivateCode, line.Presentation.Colour, line.Presentation.TextColour, routeSortOrder) + if err != nil { + return err + } + } + + err = tx.Commit() + if err != nil { + return err + } + + return nil +} From c17dd8ad0d420e443572893da54dc24900149e22 Mon Sep 17 00:00:00 2001 From: Lucie Delestre Date: Fri, 23 May 2025 23:54:18 +0200 Subject: [PATCH 11/12] =?UTF-8?q?r=C3=A9orga=20SIRI/Netex?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handlers/naolib.go | 5 +++-- services/naolib.go | 4 ++-- services/{ => naolib}/netex/decode.go | 0 services/{netex => naolib/siri}/consts.go | 2 +- services/{netex => naolib/siri}/generateXML.go | 2 +- services/{netex => naolib/siri}/request.go | 2 +- .../siri}/template/StopMonitoringRequest.xml.template | 0 7 files changed, 8 insertions(+), 7 deletions(-) rename services/{ => naolib}/netex/decode.go (100%) rename services/{netex => naolib/siri}/consts.go (92%) rename services/{netex => naolib/siri}/generateXML.go (98%) rename services/{netex => naolib/siri}/request.go (98%) rename services/{netex => naolib/siri}/template/StopMonitoringRequest.xml.template (100%) diff --git a/handlers/naolib.go b/handlers/naolib.go index 7eca4fa..bec2cee 100644 --- a/handlers/naolib.go +++ b/handlers/naolib.go @@ -8,7 +8,8 @@ import ( "github.com/gofiber/fiber/v2" "github.com/plugimt/transat-backend/models" "github.com/plugimt/transat-backend/services" - "github.com/plugimt/transat-backend/services/netex" + "github.com/plugimt/transat-backend/services/naolib/netex" + "github.com/plugimt/transat-backend/services/naolib/siri" ) const ( @@ -150,7 +151,7 @@ func (h *NaolibHandler) SearchStopPlace(c *fiber.Ctx) error { func (h *NaolibHandler) GenerateNetexRequest(c *fiber.Ctx) error { stops := []string{"CTRE2", "CTRE4"} - request, err := netex.GenerateStopMonitoringRequest(stops) + request, err := siri.GenerateStopMonitoringRequest(stops) if err != nil { return c.Status(fiber.StatusInternalServerError).SendString(err.Error() + "\n") } diff --git a/services/naolib.go b/services/naolib.go index 1447421..beef6dc 100644 --- a/services/naolib.go +++ b/services/naolib.go @@ -7,7 +7,7 @@ import ( "time" "github.com/plugimt/transat-backend/models" - "github.com/plugimt/transat-backend/services/netex" + "github.com/plugimt/transat-backend/services/naolib/siri" ) var httpClient = &http.Client{ @@ -42,7 +42,7 @@ func (s *NaolibService) GetDepartures(stopPlaceId string) (map[string]models.Dep quays = append(quays, quay) } - siriResponse, err := netex.CallStopMonitoringRequest(quays) + siriResponse, err := siri.CallStopMonitoringRequest(quays) if err != nil { return nil, err } diff --git a/services/netex/decode.go b/services/naolib/netex/decode.go similarity index 100% rename from services/netex/decode.go rename to services/naolib/netex/decode.go diff --git a/services/netex/consts.go b/services/naolib/siri/consts.go similarity index 92% rename from services/netex/consts.go rename to services/naolib/siri/consts.go index c9e200d..7fc6b8b 100644 --- a/services/netex/consts.go +++ b/services/naolib/siri/consts.go @@ -1,4 +1,4 @@ -package netex +package siri const ( RequestorRef = "opendata" diff --git a/services/netex/generateXML.go b/services/naolib/siri/generateXML.go similarity index 98% rename from services/netex/generateXML.go rename to services/naolib/siri/generateXML.go index 76e8215..e1b93c4 100644 --- a/services/netex/generateXML.go +++ b/services/naolib/siri/generateXML.go @@ -1,4 +1,4 @@ -package netex +package siri import ( "bytes" diff --git a/services/netex/request.go b/services/naolib/siri/request.go similarity index 98% rename from services/netex/request.go rename to services/naolib/siri/request.go index 9e42bc4..5a8ce0d 100644 --- a/services/netex/request.go +++ b/services/naolib/siri/request.go @@ -1,4 +1,4 @@ -package netex +package siri import ( "bytes" diff --git a/services/netex/template/StopMonitoringRequest.xml.template b/services/naolib/siri/template/StopMonitoringRequest.xml.template similarity index 100% rename from services/netex/template/StopMonitoringRequest.xml.template rename to services/naolib/siri/template/StopMonitoringRequest.xml.template From af143d1bc7efcf87e373b44579096b89b6c7b37f Mon Sep 17 00:00:00 2001 From: Lucie Delestre Date: Sat, 24 May 2025 00:10:10 +0200 Subject: [PATCH 12/12] =?UTF-8?q?Ajout=20information=20ligne=20dans=20les?= =?UTF-8?q?=20prochains=20d=C3=A9parts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- handlers/naolib.go | 27 +++++++++++++-------------- main.go | 7 +++++-- models/naolib.go | 10 ++++++++++ models/netex.go | 4 ++-- routes/naolib_routes.go | 5 +++-- services/naolib.go | 27 ++++++++++++++++++--------- services/naolib/netex/decode.go | 32 ++++++++++++++++++-------------- services/naolib/netex/lines.go | 33 +++++++++++++++++++++++++++++++++ 8 files changed, 102 insertions(+), 43 deletions(-) create mode 100644 services/naolib/netex/lines.go diff --git a/handlers/naolib.go b/handlers/naolib.go index bec2cee..2e05e79 100644 --- a/handlers/naolib.go +++ b/handlers/naolib.go @@ -2,7 +2,6 @@ package handlers import ( "database/sql" - "fmt" "os" "github.com/gofiber/fiber/v2" @@ -17,14 +16,16 @@ const ( ) type NaolibHandler struct { - service *services.NaolibService - db *sql.DB + service *services.NaolibService + netexService *netex.NetexService + db *sql.DB } -func NewNaolibHandler(service *services.NaolibService, db *sql.DB) *NaolibHandler { +func NewNaolibHandler(service *services.NaolibService, netexService *netex.NetexService, db *sql.DB) *NaolibHandler { return &NaolibHandler{ - service: service, - db: db, + service: service, + netexService: netexService, + db: db, } } @@ -52,20 +53,18 @@ func (h *NaolibHandler) ImportNetexOffer(c *fiber.Ctx) error { return c.Status(fiber.StatusBadRequest).SendString("URL is required") } - fileName, err := netex.DownloadAndExtractIfNeededOffer(url) + fileName, err := h.netexService.DownloadAndExtractIfNeededOffer(url) if err != nil { return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) } defer os.Remove(fileName) - netexData, err := netex.DecodeNetexOfferData(fileName) + netexData, err := h.netexService.DecodeNetexOfferData(fileName) if err != nil { return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) } - fmt.Println(netexData) - - err = netex.SaveNetexOfferToDatabase(netexData, h.db) + err = h.netexService.SaveNetexOfferToDatabase(netexData) if err != nil { return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) } @@ -90,18 +89,18 @@ func (h *NaolibHandler) ImportNetexStops(c *fiber.Ctx) error { return c.Status(fiber.StatusBadRequest).SendString("URL is required") } - fileName, err := netex.DownloadAndExtractIfNeeded(url) + fileName, err := h.netexService.DownloadAndExtractIfNeeded(url) if err != nil { return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) } defer os.Remove(fileName) - netexData, err := netex.DecodeNetexStopsData(fileName) + netexData, err := h.netexService.DecodeNetexStopsData(fileName) if err != nil { return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) } - err = netex.SaveNetexStopsToDatabase(netexData, h.db) + err = h.netexService.SaveNetexStopsToDatabase(netexData) if err != nil { return c.Status(fiber.StatusInternalServerError).SendString(err.Error()) } diff --git a/main.go b/main.go index 1788bb6..471d1cb 100644 --- a/main.go +++ b/main.go @@ -18,6 +18,7 @@ import ( "github.com/plugimt/transat-backend/routes" "github.com/plugimt/transat-backend/scheduler" // Import our scheduler package "github.com/plugimt/transat-backend/services" + "github.com/plugimt/transat-backend/services/naolib/netex" "github.com/plugimt/transat-backend/utils" "github.com/robfig/cron/v3" @@ -119,7 +120,9 @@ func main() { // Initialize Handlers that need explicit instantiation (e.g., for Cron) restHandler := restaurantHandler.NewRestaurantHandler(db, translationService, notificationService) - naolibService := services.NewNaolibService(db) +// Naolib + netexService := netex.NewNetexService(db) + naolibService := services.NewNaolibService(db, netexService) // Initialize Weather Service and Handler weatherService, err := services.NewWeatherService() @@ -199,7 +202,7 @@ func main() { routes.SetupWashingMachineRoutes(api) // Setup washing machine routes routes.SetupWeatherRoutes(api, weatherHandler) // Setup weather routes routes.SetupNotificationRoutes(api, db, notificationService) // Setup notification test routes - routes.SetupNaolibRoutes(api, naolibService, db) + routes.SetupNaolibRoutes(api, naolibService, netexService, db) app.Get("/health", func(c *fiber.Ctx) error { return c.SendString("OK") diff --git a/models/naolib.go b/models/naolib.go index 6e3f294..371f908 100644 --- a/models/naolib.go +++ b/models/naolib.go @@ -13,6 +13,7 @@ type Departures struct { } type Departure struct { + Line Line `json:"line"` LineRef string `json:"lineRef"` Direction string `json:"direction"` DestinationName string `json:"destinationName"` @@ -20,3 +21,12 @@ type Departure struct { ArrivalTime time.Time `json:"arrivalTime"` VehicleMode string `json:"vehicleMode"` } + +type Line struct { + ID string `json:"id"` + Name string `json:"name"` + TransportMode string `json:"transportMode"` + Number string `json:"number"` + BackgroundColour string `json:"backgroundColour"` + ForegroundColour string `json:"foregroundColour"` +} diff --git a/models/netex.go b/models/netex.go index 21d60e1..a3fc663 100644 --- a/models/netex.go +++ b/models/netex.go @@ -120,7 +120,7 @@ type GeneralFrameOffer struct { TypeOfFrameRef TypeOfFrameRef `xml:"TypeOfFrameRef"` Members struct { Network Network `xml:"Network"` - Lines []Line `xml:"Line"` + Lines []SIRILine `xml:"Line"` Operators []Operator `xml:"Operator"` Authorities []Authority `xml:"Authority"` } `xml:"members"` @@ -144,7 +144,7 @@ type AuthorityRef struct { Version string `xml:"version,attr,omitempty"` } -type Line struct { +type SIRILine struct { ID string `xml:"id,attr"` Version string `xml:"version,attr"` KeyList []KeyValue `xml:"keyList>KeyValue"` diff --git a/routes/naolib_routes.go b/routes/naolib_routes.go index ae6f4cb..dbdd740 100644 --- a/routes/naolib_routes.go +++ b/routes/naolib_routes.go @@ -6,12 +6,13 @@ import ( "github.com/gofiber/fiber/v2" "github.com/plugimt/transat-backend/handlers" "github.com/plugimt/transat-backend/services" + "github.com/plugimt/transat-backend/services/naolib/netex" ) -func SetupNaolibRoutes(router fiber.Router, naolibService *services.NaolibService, db *sql.DB) { +func SetupNaolibRoutes(router fiber.Router, naolibService *services.NaolibService, netexService *netex.NetexService, db *sql.DB) { // Groupe de routes pour Naolib naolib := router.Group("/naolib") - handler := handlers.NewNaolibHandler(naolibService, db) + handler := handlers.NewNaolibHandler(naolibService, netexService, db) // Route pour obtenir les prochains départs naolib.Get("/departures/chantrerie", handler.GetNextDeparturesChantrerie) diff --git a/services/naolib.go b/services/naolib.go index beef6dc..e0c90eb 100644 --- a/services/naolib.go +++ b/services/naolib.go @@ -2,25 +2,22 @@ package services import ( "database/sql" - "net/http" "strings" - "time" "github.com/plugimt/transat-backend/models" + "github.com/plugimt/transat-backend/services/naolib/netex" "github.com/plugimt/transat-backend/services/naolib/siri" ) -var httpClient = &http.Client{ - Timeout: 10 * time.Second, -} - type NaolibService struct { - db *sql.DB + db *sql.DB + netexService *netex.NetexService } -func NewNaolibService(db *sql.DB) *NaolibService { +func NewNaolibService(db *sql.DB, netexService *netex.NetexService) *NaolibService { return &NaolibService{ - db: db, + db: db, + netexService: netexService, } } @@ -79,7 +76,13 @@ func (s *NaolibService) GetDepartures(stopPlaceId string) (map[string]models.Dep } } + line, err := s.netexService.GetLine(lineRef) + if err != nil { + return nil, err + } + departure := models.Departure{ + Line: *line, LineRef: lineRef, Direction: lineDepartures.DepartureDirectionAller.Direction, DestinationName: departure.MonitoredVehicleJourney.DestinationName, @@ -98,7 +101,13 @@ func (s *NaolibService) GetDepartures(stopPlaceId string) (map[string]models.Dep } } + line, err := s.netexService.GetLine(lineRef) + if err != nil { + return nil, err + } + departure := models.Departure{ + Line: *line, LineRef: lineRef, Direction: lineDepartures.DepartureDirectionRetour.Direction, DestinationName: departure.MonitoredVehicleJourney.DestinationName, diff --git a/services/naolib/netex/decode.go b/services/naolib/netex/decode.go index b97226b..54a9774 100644 --- a/services/naolib/netex/decode.go +++ b/services/naolib/netex/decode.go @@ -18,7 +18,17 @@ import ( "github.com/plugimt/transat-backend/utils" ) -func DownloadAndExtractIfNeeded(url string) (string, error) { +type NetexService struct { + db *sql.DB +} + +func NewNetexService(db *sql.DB) *NetexService { + return &NetexService{ + db: db, + } +} + +func (n *NetexService) DownloadAndExtractIfNeeded(url string) (string, error) { // download the file resp, err := http.Get(url) if err != nil { @@ -104,7 +114,7 @@ func DownloadAndExtractIfNeeded(url string) (string, error) { return fileName, nil } -func DownloadAndExtractIfNeededOffer(url string) (string, error) { +func (n *NetexService) DownloadAndExtractIfNeededOffer(url string) (string, error) { // download the file resp, err := http.Get(url) if err != nil { @@ -156,14 +166,10 @@ func DownloadAndExtractIfNeededOffer(url string) (string, error) { continue } - fmt.Println("unzipping file ", filePath) - if !strings.HasPrefix(filePath, filepath.Clean(dst)+string(os.PathSeparator)) { - fmt.Println("invalid file path") return "", fmt.Errorf("invalid file path") } if f.FileInfo().IsDir() { - fmt.Println("creating directory...") os.MkdirAll(filePath, os.ModePerm) continue } @@ -172,7 +178,6 @@ func DownloadAndExtractIfNeededOffer(url string) (string, error) { panic(err) } - fmt.Println("opening file ", filePath) dstFile, err := os.OpenFile(filePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) if err != nil { panic(err) @@ -183,7 +188,6 @@ func DownloadAndExtractIfNeededOffer(url string) (string, error) { panic(err) } - fmt.Println("writing file ", filePath) if _, err := io.Copy(dstFile, fileInArchive); err != nil { panic(err) } @@ -200,7 +204,7 @@ func DownloadAndExtractIfNeededOffer(url string) (string, error) { return communFileName, nil } -func DecodeNetexStopsData(file string) (*models.PublicationDelivery, error) { +func (n *NetexService) DecodeNetexStopsData(file string) (*models.PublicationDelivery, error) { data, err := os.ReadFile(file) if err != nil { return nil, err @@ -215,7 +219,7 @@ func DecodeNetexStopsData(file string) (*models.PublicationDelivery, error) { return &netexData, nil } -func DecodeNetexOfferData(fileName string) (*models.NETEXCommonFile, error) { +func (n *NetexService) DecodeNetexOfferData(fileName string) (*models.NETEXCommonFile, error) { data, err := os.ReadFile(fileName) if err != nil { return nil, err @@ -230,12 +234,12 @@ func DecodeNetexOfferData(fileName string) (*models.NETEXCommonFile, error) { return &netexData, nil } -func SaveNetexStopsToDatabase(netexData *models.PublicationDelivery, db *sql.DB) error { +func (n *NetexService) SaveNetexStopsToDatabase(netexData *models.PublicationDelivery) error { stopPlaces := netexData.DataObjects.GeneralFrame.Members.StopPlaces quays := netexData.DataObjects.GeneralFrame.Members.Quays // on crée une transaction et on supprime toutes les données existantes, avant de les insérer - tx, err := db.Begin() + tx, err := n.db.Begin() if err != nil { return err } @@ -288,11 +292,11 @@ func SaveNetexStopsToDatabase(netexData *models.PublicationDelivery, db *sql.DB) return nil } -func SaveNetexOfferToDatabase(netexData *models.NETEXCommonFile, db *sql.DB) error { +func (n *NetexService) SaveNetexOfferToDatabase(netexData *models.NETEXCommonFile) error { lines := netexData.DataObjects.GeneralFrame.Members.Lines // on crée une transaction et on supprime toutes les données existantes, avant de les insérer - tx, err := db.Begin() + tx, err := n.db.Begin() if err != nil { return err } diff --git a/services/naolib/netex/lines.go b/services/naolib/netex/lines.go new file mode 100644 index 0000000..1424c28 --- /dev/null +++ b/services/naolib/netex/lines.go @@ -0,0 +1,33 @@ +package netex + +import "github.com/plugimt/transat-backend/models" + +func (n *NetexService) GetLines() ([]models.Line, error) { + lines := []models.Line{} + + rows, err := n.db.Query("SELECT id, name, transport_mode, public_code, colour, text_colour FROM NETEX_Line ORDER BY route_sort_order") + if err != nil { + return nil, err + } + defer rows.Close() + + for rows.Next() { + var line models.Line + err := rows.Scan(&line.ID, &line.Name, &line.TransportMode, &line.Number, &line.BackgroundColour, &line.ForegroundColour) + if err != nil { + return nil, err + } + lines = append(lines, line) + } + + return lines, nil +} + +func (n *NetexService) GetLine(id string) (*models.Line, error) { + var line models.Line + err := n.db.QueryRow("SELECT id, name, transport_mode, public_code, colour, text_colour FROM NETEX_Line WHERE id = $1", id).Scan(&line.ID, &line.Name, &line.TransportMode, &line.Number, &line.BackgroundColour, &line.ForegroundColour) + if err != nil { + return nil, err + } + return &line, nil +}