From bae019137f9620026be2adaa2d15f131731af416 Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 10:26:18 +0200 Subject: [PATCH 01/18] fix(woocommerce): expose response headers from inline WC client Adds getHeaders() / getHeader() accessors to the inline WCResponse class and captures HTTP response headers case-insensitively via CURLOPT_HEADERFUNCTION. Required foundation for pagination handling (Issue #262). --- www/pages/shopimporter_woocommerce.php | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/www/pages/shopimporter_woocommerce.php b/www/pages/shopimporter_woocommerce.php index 0d221f317..7480318f0 100644 --- a/www/pages/shopimporter_woocommerce.php +++ b/www/pages/shopimporter_woocommerce.php @@ -1271,13 +1271,26 @@ public function getCode() /** * Get headers. * - * @return array $headers WCResponse headers. + * @return array $headers WCResponse headers (keys normalized to lowercase). */ public function getHeaders() { return $this->headers; } + /** + * Get a single response header by name (case-insensitive). + * + * @param string $name Header name (e.g. 'x-wp-totalpages'). + * + * @return string|null Header value or null if not present. + */ + public function getHeader($name) + { + $key = strtolower($name); + return isset($this->headers[$key]) ? $this->headers[$key] : null; + } + /** * Get body. * @@ -2163,6 +2176,7 @@ protected function getResponseHeaders() list($key, $value) = explode(': ', $line); + $key = strtolower($key); $headers[$key] = isset($headers[$key]) ? $headers[$key] . ', ' . trim($value) : trim($value); } From 1907cdf07ee3eb51f88ae32f66cf4f663c3de9b9 Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 10:31:27 +0200 Subject: [PATCH 02/18] fix(woocommerce): add getLastResponse() accessor on WC client Exposes the underlying WCResponse of the most recent request so callers can read response headers (X-WP-Total, X-WP-TotalPages) without changing the existing JSON-body return contract. Follow-up to 291197d, required by the pagination work in issue #262. --- www/pages/shopimporter_woocommerce.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/www/pages/shopimporter_woocommerce.php b/www/pages/shopimporter_woocommerce.php index 7480318f0..9908b83b0 100644 --- a/www/pages/shopimporter_woocommerce.php +++ b/www/pages/shopimporter_woocommerce.php @@ -1189,6 +1189,16 @@ public function options($endpoint) { return $this->http->request($endpoint, 'OPTIONS'); } + + /** + * Get the WCResponse from the most recent HTTP request. + * + * @return WCResponse|null + */ + public function getLastResponse() + { + return $this->http->getResponse(); + } } class WCResponse From dee30b17ce81a7c382b475e87e45553d3b196444 Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 10:37:22 +0200 Subject: [PATCH 03/18] fix(woocommerce): persist last-import timestamp in shopexport config Reads felder.letzter_import_timestamp from shopexport.einstellungen_json with a 30-day fallback for first runs, and adds a persistLastImportTimestamp() helper that does a read-modify-write via DatabaseService named params. Infrastructure for the pagination loop in issue #262; not yet called here. --- www/pages/shopimporter_woocommerce.php | 41 ++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/www/pages/shopimporter_woocommerce.php b/www/pages/shopimporter_woocommerce.php index 9908b83b0..c68667771 100644 --- a/www/pages/shopimporter_woocommerce.php +++ b/www/pages/shopimporter_woocommerce.php @@ -60,6 +60,12 @@ class Shopimporter_Woocommerce extends ShopimporterBase /** @var Logger $logger */ public $logger; + /** @var bool $ssl_ignore Whether to ignore SSL certificate validation */ + public $ssl_ignore; + + /** @var string $lastImportTimestamp ISO-8601 UTC timestamp of the last successful import */ + public $lastImportTimestamp; + public function __construct($app, $intern = false) { $this->app = $app; @@ -891,6 +897,41 @@ public function getKonfig($shopid, $data) $this->ssl_ignore ); + $storedTimestamp = $preferences['felder']['letzter_import_timestamp'] ?? null; + if (!empty($storedTimestamp)) { + $this->lastImportTimestamp = $storedTimestamp; + } else { + $this->lastImportTimestamp = gmdate('Y-m-d\TH:i:s', strtotime('-30 days')); + } + + } + + /** + * Persists the last successful import timestamp to shopexport.einstellungen_json. + * Does a read-modify-write to preserve all other fields. + * + * @param string $isoUtcDate ISO-8601 UTC timestamp, e.g. '2026-04-20T12:34:56' + * @return void + */ + public function persistLastImportTimestamp($isoUtcDate) + { + $einstellungen_json = $this->app->DatabaseService->selectValue( + "SELECT einstellungen_json FROM shopexport WHERE id = :id LIMIT 1", + ['id' => (int)$this->shopid] + ); + $einstellungen = []; + if (!empty($einstellungen_json)) { + $einstellungen = json_decode($einstellungen_json, true) ?: []; + } + if (!isset($einstellungen['felder']) || !is_array($einstellungen['felder'])) { + $einstellungen['felder'] = []; + } + $einstellungen['felder']['letzter_import_timestamp'] = $isoUtcDate; + $this->app->DatabaseService->execute( + "UPDATE shopexport SET einstellungen_json = :json WHERE id = :id", + ['json' => json_encode($einstellungen), 'id' => (int)$this->shopid] + ); + $this->lastImportTimestamp = $isoUtcDate; } /** From ec9a6176b2dd730cde3f37e10c1716f65c7b0c0a Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 10:45:53 +0200 Subject: [PATCH 04/18] fix(woocommerce): use after-filter and pagination for order import Replaces the fake greater-than-id filter (800 hardcoded IDs) with the WC v3 after= parameter and walks X-WP-TotalPages up to MAX_PAGES_PER_RUN=5 pages per run (500 orders). Persists a progress timestamp via persistLastImportTimestamp() after each processed order so aborted runs resume cleanly. Adds a one-shot ab_nummer->timestamp translation for existing shops transitioning from the legacy cursor. Fixes silent data loss when more than 20 orders arrived between runs. Issue #262. --- www/pages/shopimporter_woocommerce.php | 216 ++++++++++++++----------- 1 file changed, 119 insertions(+), 97 deletions(-) diff --git a/www/pages/shopimporter_woocommerce.php b/www/pages/shopimporter_woocommerce.php index c68667771..f2106fe56 100644 --- a/www/pages/shopimporter_woocommerce.php +++ b/www/pages/shopimporter_woocommerce.php @@ -66,6 +66,12 @@ class Shopimporter_Woocommerce extends ShopimporterBase /** @var string $lastImportTimestamp ISO-8601 UTC timestamp of the last successful import */ public $lastImportTimestamp; + /** @var bool $lastImportTimestampIsFallback True when lastImportTimestamp was computed as 30-day fallback */ + public $lastImportTimestampIsFallback = false; + + const MAX_PAGES_PER_RUN = 5; + const ORDERS_PER_PAGE = 100; + public function __construct($app, $intern = false) { $this->app = $app; @@ -81,135 +87,149 @@ public function ImportList() } /** - * This function returns the number of orders which have not yet been imported + * Returns the total number of orders pending import since the last import + * timestamp. Uses the WC v3 after= parameter and reads the count from + * the X-WP-Total response header (per_page=1 to minimise payload). + * + * @return int */ public function ImportGetAuftraegeAnzahl() { - // Query the API to get new orders, filtered by the order status as specifed by the user. - // We set per_page to 100 - this could lead to a situation where there are more than - // 100 new Orders, but we still only return 100. - - // Array containing additional settings, namely 'ab_nummer' (containting the next order number to get) - // and 'holeallestati' (an integer) - $tmp = $this->CatchRemoteCommand('data'); - - // Only orders having an order number greater or equal than this should be fetched. null otherwise - $number_from = empty($tmp['ab_nummer']) ? null : (int) $tmp['ab_nummer']; - - // pending orders will be fetched into this array. it's length is returned at the end of the funciton - $pendingOrders = array(); + $configuredStatuses = array_map('trim', explode(';', $this->statusPending)); - if ($number_from) { - // Number-based import is selected - - // The WooCommerce API doenst allow for a proper "greater than id n" request. - // we fake this behavior by creating an array that contains 'many' (~ 1000) consecutive - // ids that are greater than $from_number and use this array with the 'include' property - // of the WooCommerce API - - $number_to = $number_from + 800; - if (!empty($tmp['bis_nummer'])) { - $number_to = $tmp['bis_nummer']; - } - - $fakeGreaterThanIds = range($number_from, $number_to); - - $pendingOrders = $this->client->get('orders', [ - 'per_page' => 100, - 'include' => implode(",", $fakeGreaterThanIds), + try { + $this->client->get('orders', [ + 'status' => $configuredStatuses, + 'after' => $this->lastImportTimestamp, + 'per_page' => 1, ]); + } catch (Exception $e) { + $this->logger->warning('WooCommerce ImportGetAuftraegeAnzahl: API request failed: ' . $e->getMessage()); + return 0; + } - } else { - // fetch posts by status - - $pendingOrders = $this->client->get('orders', [ - 'status' => array_map('trim', explode(';', $this->statusPending)), - 'per_page' => 100 - ]); + $wcResponse = $this->client->getLastResponse(); + if ($wcResponse === null) { + $this->logger->warning('WooCommerce ImportGetAuftraegeAnzahl: getLastResponse() returned null'); + return 0; + } + $total = $wcResponse->getHeader('x-wp-total'); + if ($total === null) { + $this->logger->warning('WooCommerce ImportGetAuftraegeAnzahl: X-WP-Total header missing'); + return 0; } - return (!empty($pendingOrders) ? count($pendingOrders) : 0); + return (int) $total; } /** - * Calling this function queries the api for pending orders and returns them - * as an array. + * Queries the WooCommerce API for pending orders since the last import + * timestamp and returns them as a Xentral-formatted array. + * + * Uses the WC v3 after= filter and paginates up to MAX_PAGES_PER_RUN + * pages (500 orders max per run). Progress timestamp is persisted after + * each processed order so aborted runs resume cleanly. * - * TODO: Only one single order is returned per invocation of this function. - * Given that we have to perform an exteremly expensive external HTTP call - * every time we call this function and could easily process more than one - * order this seems very bad performance-wise. + * @return array|null */ public function ImportGetAuftrag() { - // Array containing additional settings, namely 'ab_nummer' (containting the next order number to get) - // and 'holeallestati' (an integer) - $tmp = $this->CatchRemoteCommand('data'); + $data = $this->CatchRemoteCommand('data'); - // Only orders having an order number greater or equal than this should be fetched. null otherwise - $number_from = empty($tmp['ab_nummer']) ? null : (int) $tmp['ab_nummer']; + // Transition: if timestamp is still a fallback and a legacy ab_nummer + // cursor is present, resolve it to a timestamp once. + if ($this->lastImportTimestampIsFallback && !empty($data['ab_nummer'])) { + $resolved = $this->resolveAbNummerToTimestamp((int) $data['ab_nummer']); + if ($resolved !== null) { + $this->persistLastImportTimestamp($resolved); + } + // lastImportTimestampIsFallback stays true only if resolution failed; + // persistLastImportTimestamp() updated $this->lastImportTimestamp already. + } - // pending orders will be fetched into this array. it's length is returned at the end of the funciton - $pendingOrders = array(); + $configuredStatuses = array_map('trim', explode(';', $this->statusPending)); + $page = 1; + $result = []; + + while ($page <= self::MAX_PAGES_PER_RUN) { + try { + $pageOrders = $this->client->get('orders', [ + 'status' => $configuredStatuses, + 'after' => $this->lastImportTimestamp, + 'per_page' => self::ORDERS_PER_PAGE, + 'page' => $page, + 'orderby' => 'date', + 'order' => 'asc', + ]); + } catch (Exception $e) { + $this->logger->warning('WooCommerce ImportGetAuftrag page=' . $page . ': ' . $e->getMessage()); + break; + } - if ($number_from) { - // Number-based import is selected + if (empty($pageOrders)) { + break; + } - // The WooCommerce API doenst allow for a proper "greater than id n" request. - // we fake this behavior by creating an array that contains 'many' (~ 1000) consecutive - // ids that are greater than $from_number and use this array with the 'include' property - // of the WooCommerce API + foreach ($pageOrders as $wcOrder) { + if (is_null($wcOrder)) { + continue; + } - $number_to = $number_from + 800; - if (!empty($tmp['bis_nummer'])) { - $number_to = $tmp['bis_nummer']; - } + $order = $this->parseOrder($wcOrder); - $fakeGreaterThanIds = range($number_from, $number_to); + $result[] = [ + 'id' => $order['auftrag'], + 'sessionid' => '', + 'logdatei' => '', + 'warenkorb' => base64_encode(serialize($order)), + ]; - $pendingOrders = $this->client->get('orders', [ - 'per_page' => 20, - 'include' => implode(',', $fakeGreaterThanIds), - 'order' => 'asc', - 'orderby' => 'id' - ]); + // Persist progress after every order so partial runs resume cleanly. + if (!empty($wcOrder->date_created_gmt)) { + $this->persistLastImportTimestamp((string) $wcOrder->date_created_gmt); + } + } - } else { - // fetch posts by status + $wcResponse = $this->client->getLastResponse(); + if ($wcResponse === null) { + break; + } - $pendingOrders = $this->client->get('orders', [ - 'status' => array_map('trim', explode(';', $this->statusPending)), - 'per_page' => 20, - 'order' => 'asc', - 'orderby' => 'id' - ]); + $totalPages = (int) $wcResponse->getHeader('x-wp-totalpages'); + if ($totalPages < 1 || $page >= $totalPages) { + break; + } + $page++; } - // Return an empty array in case there are no orders to import - if ((!empty($pendingOrders) ? count($pendingOrders) : 0) === 0) { + return !empty($result) ? $result : null; + } + + /** + * Resolves a legacy WooCommerce order ID (ab_nummer) to the order's + * date_created_gmt timestamp for the one-shot transition from cursor- + * based to timestamp-based import. + * + * @param int $abNummer WooCommerce order ID + * @return string|null ISO-8601 UTC timestamp or null on failure + */ + private function resolveAbNummerToTimestamp($abNummer) + { + try { + $order = $this->client->get('orders/' . $abNummer); + } catch (Exception $e) { + $this->logger->warning('WooCommerce resolveAbNummerToTimestamp(' . $abNummer . '): ' . $e->getMessage()); return null; } - $tmp = []; - - foreach ($pendingOrders as $pendingOrder) { - $wcOrder = $pendingOrder; - $order = $this->parseOrder($wcOrder); - - if (is_null($wcOrder)) { - continue; - } - - $tmp[] = [ - 'id' => $order['auftrag'], - 'sessionid' => '', - 'logdatei' => '', - 'warenkorb' => base64_encode(serialize($order)), - ]; + if (empty($order->date_created_gmt)) { + $this->logger->warning('WooCommerce resolveAbNummerToTimestamp(' . $abNummer . '): date_created_gmt missing'); + return null; } - return $tmp; + + return (string) $order->date_created_gmt; } // This function searches the wcOrder for the specified WC Meta key @@ -900,8 +920,10 @@ public function getKonfig($shopid, $data) $storedTimestamp = $preferences['felder']['letzter_import_timestamp'] ?? null; if (!empty($storedTimestamp)) { $this->lastImportTimestamp = $storedTimestamp; + $this->lastImportTimestampIsFallback = false; } else { $this->lastImportTimestamp = gmdate('Y-m-d\TH:i:s', strtotime('-30 days')); + $this->lastImportTimestampIsFallback = true; } } From b7d3a2710a9db82f3e9ad60011d9155d5b055644 Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 10:53:39 +0200 Subject: [PATCH 05/18] fix(woocommerce): stabilize after-filter across pagination run Captures lastImportTimestamp into a local variable before the pagination loop so progress persistence inside the loop does not mutate the GET filter. Without this, after=\$lastTs moves forward each iteration while page advances too, causing 100 orders per extra page to be skipped. Also fixes two smaller issues: - resolveAbNummerToTimestamp() returns ts-1 so the strictly-after filter does not lose the transition order. - explode(';', \$this->statusPending) is now PHP 8.1+ safe via (string) cast. Follow-up to abe58aab, addresses code review findings on issue #262. --- www/pages/shopimporter_woocommerce.php | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/www/pages/shopimporter_woocommerce.php b/www/pages/shopimporter_woocommerce.php index f2106fe56..299b83882 100644 --- a/www/pages/shopimporter_woocommerce.php +++ b/www/pages/shopimporter_woocommerce.php @@ -95,7 +95,7 @@ public function ImportList() */ public function ImportGetAuftraegeAnzahl() { - $configuredStatuses = array_map('trim', explode(';', $this->statusPending)); + $configuredStatuses = array_map('trim', explode(';', (string) $this->statusPending)); try { $this->client->get('orders', [ @@ -148,7 +148,8 @@ public function ImportGetAuftrag() // persistLastImportTimestamp() updated $this->lastImportTimestamp already. } - $configuredStatuses = array_map('trim', explode(';', $this->statusPending)); + $configuredStatuses = array_map('trim', explode(';', (string) $this->statusPending)); + $runStartTimestamp = $this->lastImportTimestamp; $page = 1; $result = []; @@ -156,7 +157,7 @@ public function ImportGetAuftrag() try { $pageOrders = $this->client->get('orders', [ 'status' => $configuredStatuses, - 'after' => $this->lastImportTimestamp, + 'after' => $runStartTimestamp, 'per_page' => self::ORDERS_PER_PAGE, 'page' => $page, 'orderby' => 'date', @@ -229,7 +230,11 @@ private function resolveAbNummerToTimestamp($abNummer) return null; } - return (string) $order->date_created_gmt; + $ts = strtotime((string) $order->date_created_gmt); + if ($ts === false) { + return null; + } + return gmdate('Y-m-d\TH:i:s', $ts - 1); } // This function searches the wcOrder for the specified WC Meta key From 810ae749ddf01c099d0756323c6c935aa946575a Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 10:56:47 +0200 Subject: [PATCH 06/18] docs: add plan for woocommerce pagination fix Captures scope, fix parameters (MAX_PAGES_PER_RUN=5, 30-day first-run fallback, UTC timestamps), implementation steps, integration test matrix T1-T10, rollout and rollback strategy, risks and mitigations. Companion doc to issue #262 and the fix commits on this branch. --- docs/plans/woocommerce-pagination-fix.md | 325 +++++++++++++++++++++++ 1 file changed, 325 insertions(+) create mode 100644 docs/plans/woocommerce-pagination-fix.md diff --git a/docs/plans/woocommerce-pagination-fix.md b/docs/plans/woocommerce-pagination-fix.md new file mode 100644 index 000000000..ce643be46 --- /dev/null +++ b/docs/plans/woocommerce-pagination-fix.md @@ -0,0 +1,325 @@ +# WooCommerce-Bestellimport: Pagination + `after`-Filter + +**Issue:** [openxe-org/openxe#262](https://github.com/OpenXE-org/OpenXE/issues/262) +**Branch:** `fix/woocommerce-order-import-pagination` +**Base:** `development` @ `8bb13973` +**Typ:** Bugfix (stiller Datenverlust) + +--- + +## 1. Ziel + +Der WooCommerce-Shopimporter verliert Bestellungen, wenn mehr als 20 neue Aufträge +zwischen zwei Cron-Läufen eingehen. Dieser Fix macht den Bestellabruf vollständig +und verlustfrei, ohne neue Plugin-Abhängigkeiten und ohne Wechsel der API-Version +(bleibt auf `wc/v3`). + +## 2. Scope + +### IN-Scope + +- Pagination-Loop beim Bestellabruf (`X-WP-TotalPages`-Header auswerten) +- `after`-Filter (ISO-8601) statt 800er-`include`-Hack +- Persistenz "letzter erfolgreicher Import-Timestamp" pro Shop +- Fallback-Logik für Erstlauf (kein Timestamp vorhanden) +- Transitions-Kompatibilität zum Legacy-Parameter `ab_nummer` + +### OUT-of-Scope (eigene Issues/PRs) + +- Batch-Endpoints fuer Stock-Sync (`products/batch`) +- Retry / Backoff fuer 429/5xx +- Composer-Migration des inline-WC-SDK +- Webhook-Support (#239) +- HPOS-Testmatrix +- UI-Reset-Button fuer den Import-Timestamp + +## 3. Fix-Parameter (final) + +| Parameter | Wert | Begruendung | +|---|---|---| +| `MAX_PAGES_PER_RUN` | **5** | Max. 500 Bestellungen pro Cron-Lauf. Bei groesseren Rueckstaenden arbeiten nachfolgende Laeufe die Restmenge ab. Verhindert Cron-Timeouts. | +| `ORDERS_PER_PAGE` | **100** | WC-v3-Default-Maximum. Minimiert Roundtrips. | +| Erstlauf-Fallback | **30 Tage** | Sinnvoller Mittelweg: holt historische Bestellungen, aber nicht unbegrenzt. UI-Override in Folge-PR moeglich. | +| Persistenz-Feld | `shopexport.einstellungen_json` → `felder.letzter_import_timestamp` | Bestehende Struktur nutzen, keine DB-Migration. | +| Timestamp-Format | ISO-8601 `Y-m-d\TH:i:s` (UTC) | Direkt an `after=` durchreichbar. | + +## 4. Datei- und Funktions-Landkarte + +Zielmodul: `www/pages/shopimporter_woocommerce.php` + +| Funktion | Zeilen | Betroffen? | Aenderung | +|---|---|---|---| +| `ImportGetAuftraegeAnzahl` | ~80–135 | ja | `include`-Hack raus, `after`-Filter rein, nur noch Count via `X-WP-Total` | +| `ImportGetAuftrag` | ~138–220 | ja | Seiten-Schleife, `after`-Filter, Timestamp-Update am Ende | +| `parseOrder` | ~222–305 | nein | unveraendert (liefert weiterhin pro Order) | +| `CatchRemoteCommand('data')` | mehrfach | mittelbar | stellt `letzter_import_timestamp` aus `einstellungen_json` bereit | +| `getKonfig` | ~802–837 | nein | nicht anfassen (separates Issue #224) | +| Inline-Client (`WCClient`, `WCHttpClient`, `WCResponse`) | 1021–2370 | ggf. ja | Header-Accessor ergaenzen falls nicht vorhanden | + +## 5. Implementierungsschritte + +### Schritt 1 — Header-Durchreichung im inline-Client pruefen + +**Aufgabe:** Klaeren, ob `WCResponse` die HTTP-Antwort-Header (`X-WP-Total`, +`X-WP-TotalPages`, `Link`) bereits als Array liefert oder ob der inline-Client +angepasst werden muss. + +**Deep-Read-Ziele (nur Lesen, kein Edit):** +- `WCHttpClient::processResponse()` / aequivalent (vermutlich ~Zeile 2140–2200) +- `WCResponse`-Constructor: werden Headers gespeichert? +- `curl_setopt`-Setup: ist `CURLOPT_HEADERFUNCTION` oder `CURLOPT_HEADER` aktiv? + +**Entscheidungs-Gate:** +- Wenn Headers bereits im Response-Objekt: direkt weiter mit Schritt 3. +- Wenn nicht: Schritt 2 vor Schritt 3. + +**Akzeptanzkriterium:** Klarer Plan, welche Client-Aenderung noetig ist (oder dass keine noetig ist). + +### Schritt 2 — Headers exponieren (bedingt) + +**Nur falls Schritt 1 ergibt, dass Headers nicht zur Businesslogik durchdringen.** + +- `WCHttpClient::processResponse()`: Header-String in assoziatives Array parsen. +- `WCResponse` um `getHeaders()` / `getHeader(string $name)` erweitern. +- cURL-Setup: `CURLOPT_HEADERFUNCTION` registrieren, Headers in Sammelarray schreiben. +- Case-insensitive Lookup (`strtolower`-normalisiert speichern), da WP Headers teils + `x-wp-totalpages` vs. `X-WP-TotalPages` sendet. + +**Akzeptanzkriterium:** `$response->getHeader('x-wp-totalpages')` liefert eine Zahl als +String (z.B. `"3"`). + +**Risiko:** Client-Klassen werden auch an anderen Stellen instanziiert (theoretisch). +→ Mit `grep` verifizieren, dass `WCClient`/`WCResponse` nur in +`shopimporter_woocommerce.php` verwendet werden (keine Referenzen in +anderen `www/pages/*.php`-Dateien). + +### Schritt 3 — Timestamp-Persistenz + +**3a — Lesen:** +- In `getKonfig()` oder `CatchRemoteCommand('data')` den Wert + `$felder['letzter_import_timestamp']` auslesen. +- Fallback: `date('Y-m-d\TH:i:s', strtotime('-30 days'))` wenn leer/null. +- Auf Klassen-Property `$this->lastImportTimestamp` ablegen. + +**3b — Schreiben:** +- Nach erfolgreichem Lauf (am Ende von `ImportGetAuftrag`): den Timestamp der + **zuletzt verarbeiteten Bestellung** (nicht `now()`) in `einstellungen_json` zurueckschreiben. +- Grund: wenn der Lauf bei Order #n+3 abbricht, beim naechsten Lauf mit Order #n+3 + weiterarbeiten, nicht mit `now()` (→ Datenverlust). +- SQL: `UPDATE shopexport SET einstellungen_json = :json WHERE id = :id`. +- Muss ueber den in `DatabaseService` vorhandenen Mechanismus laufen (named params, + Prepared Statement — siehe `CLAUDE.md`-Projektregel). + +**3c — Atomic Update:** +- Nur bei erfolgreichem Import-Ende Timestamp persistieren. +- Bei Exception mitten im Lauf: Timestamp NICHT auf den Absturzpunkt schreiben + — besser: pro erfolgreich verarbeiteter Order einzeln fortschreiben (Progress), + sodass ein Absturz nur den aktuellen, nicht alle bisherigen, verliert. + +**Akzeptanzkriterium:** +``` +SELECT einstellungen_json FROM shopexport WHERE id = +→ enthaelt 'letzter_import_timestamp': '2026-04-20T12:34:56' +``` + +### Schritt 4 — Refactor `ImportGetAuftraegeAnzahl` (Count-Funktion) + +**Alt (Zeile 80–135):** +- Query: `orders?status=…&include=<800 IDs>&per_page=100`. +- Liest `count($response)` als Return. + +**Neu:** +- Query: `orders?status[]=&status[]=&after=&per_page=1`. +- Return: `(int) $response->getHeader('x-wp-total')`. +- `per_page=1` reicht — wir brauchen nur den Count-Header, nicht die Daten. + +**Akzeptanzkriterium:** +- Bei 0 neuen Orders: liefert 0. +- Bei 250 neuen Orders: liefert 250 (nicht 100). + +### Schritt 5 — Refactor `ImportGetAuftrag` (Import-Funktion) + +**Alt (Zeile 138–220):** +- Query: `orders?status=…&include=<800 IDs>&per_page=20&orderby=id&order=asc`. +- Iteriert einmalig ueber Response, ruft `parseOrder` pro Eintrag. + +**Neu — Pseudocode-Anker (keine OpenXE-Implementierung, nur Logik):** + +``` +const MAX_PAGES_PER_RUN = 5 +const ORDERS_PER_PAGE = 100 + +lastTs = $this->lastImportTimestamp +orders = [] +page = 1 + +while page <= MAX_PAGES_PER_RUN: + response = client.get('orders', { + status[]: $configuredStatuses, + after: lastTs, + per_page: ORDERS_PER_PAGE, + page: page, + orderby: 'date', + order: 'asc', + }) + + if response is empty: + break + + for order in response: + parsed = parseOrder(order) + orders.push(parsed) + # Progress-Timestamp: nach jeder Order fortschreiben + this.persistLastImportTimestamp(order.date_created_gmt) + + totalPages = (int) response.getHeader('x-wp-totalpages') + if page >= totalPages: + break + + page++ + +return orders +``` + +**Wichtig:** +- `orderby=date` + `order=asc` garantiert, dass die **aelteste** neue Order zuerst kommt. + Dadurch kann der Progress-Timestamp monoton wachsen. +- `date_created_gmt` als Referenz (nicht `date_created` — Zeitzonen-Fallen vermeiden). +- `MAX_PAGES_PER_RUN` ist eine Konstante am Klassenanfang, nicht per-Config (kann in + Folge-PR konfigurierbar werden). + +**Akzeptanzkriterium:** +- Bei 250 neuen Orders ueber einen Lauf: alle 250 importiert. +- Bei 750 neuen Orders: Lauf 1 importiert 500, Lauf 2 importiert 250. + +### Schritt 6 — Transitions-Kompatibilitaet `ab_nummer` + +**Ist-Zustand:** `CatchRemoteCommand('data')` liefert u.a. `ab_nummer` — die naechste +Bestell-Nummer, ab der gelesen werden soll (Legacy-Cursor). + +**Uebergangsregel:** +- Wenn `letzter_import_timestamp` gesetzt → `after`-Filter nutzen, `ab_nummer` ignorieren. +- Wenn `letzter_import_timestamp` leer aber `ab_nummer` > 0 → einmalig `ab_nummer` in + einen Timestamp uebersetzen (Query `GET orders/{ab_nummer}` → `date_created_gmt` lesen), + als `letzter_import_timestamp` persistieren, ab dann `after`-Logik. +- Wenn beides leer → 30-Tage-Fallback (Schritt 3a). + +**Akzeptanzkriterium:** Shop, der bisher mit `ab_nummer` lief, importiert nach +Update **keine Duplikate** und **keine Luecken**. + +### Schritt 7 — `include`-Hack entfernen + +- Loeschen: + - Zeile ~99–113 (Count-Pfad `include`-Aufbau) + - Zeile ~153–167 (Import-Pfad `include`-Aufbau) + - Die beiden selbstkritischen Code-Kommentare *"fake"-Filter* +- Keine Ersatz-Struktur — `after` uebernimmt den Job komplett. + +### Schritt 8 — Cleanup & Commits + +**Pre-Commit-Checks:** +- `php -l www/pages/shopimporter_woocommerce.php` +- Trailing Whitespace raus (Subagent-Reste) +- CRLF-Warnungen ignorieren (autocrlf-Artefakte per Projekt-Regel) + +**Commit-Struktur (atomar, auf `fix/woocommerce-order-import-pagination`):** + +1. `fix(woocommerce): expose response headers from inline WC client` + → nur Client-Aenderung (bedingt; entfaellt wenn Schritt 1 Headers schon freigibt) + +2. `fix(woocommerce): persist last-import timestamp in shopexport config` + → Timestamp-Read/Write + 30-Tage-Fallback + Progress-Update pro Order + +3. `fix(woocommerce): use after-filter and pagination for order import` + → Kernaenderung an `ImportGetAuftrag` und `ImportGetAuftraegeAnzahl` + +4. `refactor(woocommerce): remove 800-id include hack` + → Aufraeumen toter Code + Kommentare + +5. `docs: add plan for woocommerce pagination fix` + → Diese Datei (`docs/plans/woocommerce-pagination-fix.md`) + +**PR-Ziel:** `openxe-org/openxe:master` (Upstream hat kein `development`). +**PR-Body:** Verweis auf Issue #262 + knappe Zusammenfassung der 4 Commits. + +## 6. Test-Plan (Integration gegen `192.168.0.143`) + +Keine Unit-Tests (OpenXE hat keine Test-Suite fuer Shopimporter). +Manuelle Integrationstests mit seeded Test-Orders. + +### Setup +- WP-Backend: `admin:password` +- WC-REST: `consumer_key`/`consumer_secret` generieren (Admin → WooCommerce → Einstellungen → Erweitert → REST-API) +- OpenXE-Shop-Konfiguration: bestehende Testinstanz wiederverwenden + +### Test-Matrix + +| # | Szenario | Start-State | Aktion | Erwartung | +|---|---|---|---|---| +| T1 | Frischinstall, keine Orders | `letzter_import_timestamp` leer | Lauf starten | 0 Orders, Timestamp bleibt leer | +| T2 | Frischinstall, 10 Orders <30 Tage alt | `letzter_import_timestamp` leer | Lauf starten | 10 Orders importiert, Timestamp = neueste Order | +| T3 | Frischinstall, 10 Orders >30 Tage alt | `letzter_import_timestamp` leer | Lauf starten | 0 Orders (Fallback-Schwelle), Timestamp leer | +| T4 | Standardlauf, 30 neue Orders | Timestamp gesetzt | Lauf starten | 30 Orders, Timestamp fortgeschrieben | +| T5 | Spike, 150 neue Orders | Timestamp gesetzt | Lauf starten | 150 Orders (1 Lauf, 2 Seiten), Timestamp fortgeschrieben | +| T6 | Rueckstand, 750 neue Orders | Timestamp gesetzt | Lauf 1 starten | 500 Orders importiert, `MAX_PAGES_PER_RUN`-Cap greift | +| T6b | Rueckstand Teil 2 | nach T6 | Lauf 2 starten | restliche 250 Orders, Timestamp final | +| T7 | Transition, Shop mit `ab_nummer`=12345, Timestamp leer | `ab_nummer=12345` | Lauf starten | `ab_nummer` in Timestamp uebersetzt, keine Duplikate | +| T8 | Abbruch mitten im Lauf | 50 von 100 verarbeitet, dann Exception | neuer Lauf | wieder bei Order 51 starten, nicht doppelt importiert | +| T9 | Idempotenz | T4 erfolgreich gelaufen | T4 nochmal starten | 0 neue Orders, keine Duplikate | +| T10 | URL-Laenge (Regression) | 800 alte Orders existieren | Lauf starten | URL bleibt kurz (<2 KB), keine `include`-Liste mehr | + +### Mess-Artefakte (vor/nach) + +- Anzahl HTTP-Requests pro Lauf (via Apache-Log auf Testinstanz) +- Laenge der URL im `GET /orders`-Request +- Laufzeit pro Lauf +- Anzahl importierte Orders pro Lauf + +## 7. Rollout & Rueckwaerts-Kompatibilitaet + +### Migration beim Update +- Keine DB-Migration noetig. +- Beim ersten Lauf nach Update: + - Wenn `ab_nummer` gesetzt → einmalige Uebersetzung (Schritt 6). + - Wenn nichts gesetzt → 30-Tage-Fallback. +- Kein Breaking Change fuer bestehende Shops. + +### Rollback-Szenario +- Rueckkehr zum alten Verhalten: Revert der 4 Commits aus Schritt 8. +- `letzter_import_timestamp` in `einstellungen_json` stoert alte Version nicht + (unbekannte Keys werden ignoriert). + +### Kompatibilitaet mit anderen Modulen +- `class.remote.php` → nicht beruehrt. +- Andere `shopimporter_*`-Module → nicht beruehrt (nur WooCommerce-spezifisch). +- `class.erpapi.php` → nicht beruehrt. + +## 8. Risiken & Mitigations + +| Risiko | Wahrscheinlichkeit | Auswirkung | Mitigation | +|---|---|---|---| +| Inline-Client liefert Headers nicht durch | mittel | hoch (blockiert Schritt 4/5) | Schritt 1 als Entscheidungs-Gate, Schritt 2 bedingter Vorarbeits-Schritt | +| Shop-Timezone vs. `after`-Zeitzone | mittel | mittel (Off-by-one-Tag) | `date_created_gmt` und `after` beide in UTC, explizit testen (T4, T9) | +| Kunden ohne Timestamp + >30 Tage alte neue Orders | niedrig | mittel | Dokumentiert in PR-Body, UI-Override in Folge-PR | +| `orderby=date` Ordering nicht deterministisch bei gleichem Timestamp | niedrig | niedrig | Zusaetzlich `orderby=date&order=asc` und WC liefert stabile Sekundaersortierung nach ID | +| Andere Shopimporter erben Basisklasse-Struktur | niedrig | niedrig | Nur `shopimporter_woocommerce.php` anfassen, `ShopimporterBase` nicht anruehren | +| URL-Laengenlimits der WAF beim `status[]`-Array | sehr niedrig | niedrig | Typisch 2-3 Status-Werte, URL bleibt kurz | + +## 9. Definition of Done + +- [ ] Schritt 1 abgeschlossen, Entscheidung fuer/gegen Schritt 2 dokumentiert im PR-Body +- [ ] Alle geplanten Commits auf `fix/woocommerce-order-import-pagination` +- [ ] `php -l` ohne Fehler +- [ ] Testmatrix T1–T10 auf `192.168.0.143` durchgelaufen, Ergebnisse im PR-Body +- [ ] Issue #262 im PR referenziert (`Fixes #262`) +- [ ] PR gegen `openxe-org/openxe:master` eroeffnet +- [ ] Diese Plan-Datei mitversioniert im Fix-Branch + +## 10. Referenzen + +- Issue: https://github.com/OpenXE-org/OpenXE/issues/262 +- WC REST API Docs (Orders): https://woocommerce.github.io/woocommerce-rest-api-docs/#orders +- WP-REST-API Pagination: https://developer.wordpress.org/rest-api/using-the-rest-api/pagination/ +- Parallele Issues: + - #239 Webhook Support (push statt poll) + - #224 JSON-Error in `getKonfig` From 7a4ee0b9791f1b9e8a5693fda4c59ae0152d5039 Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 17:04:13 +0200 Subject: [PATCH 07/18] fix(woocommerce): CLI-context fallback for persistLastImportTimestamp $this->app->DatabaseService is only lazy-bound in the web context. When the shopimporter runs through the cron trigger the service is not available, which breaks the timestamp persistence path. Falls back to $this->app->DB with real_escape_string when DatabaseService is absent. Discovered during the WC 8.9.3 + 10.7.0 integration test matrix on the .143 test instance. --- www/pages/shopimporter_woocommerce.php | 37 +++++++++++++++++++------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/www/pages/shopimporter_woocommerce.php b/www/pages/shopimporter_woocommerce.php index 299b83882..bfcdb0f10 100644 --- a/www/pages/shopimporter_woocommerce.php +++ b/www/pages/shopimporter_woocommerce.php @@ -942,10 +942,19 @@ public function getKonfig($shopid, $data) */ public function persistLastImportTimestamp($isoUtcDate) { - $einstellungen_json = $this->app->DatabaseService->selectValue( - "SELECT einstellungen_json FROM shopexport WHERE id = :id LIMIT 1", - ['id' => (int)$this->shopid] - ); + $shopid = (int)$this->shopid; + // Prefer DatabaseService when available (web context), fall back to DB + // so this method also works in the CLI/cron context. + if (!empty($this->app->DatabaseService)) { + $einstellungen_json = $this->app->DatabaseService->selectValue( + "SELECT einstellungen_json FROM shopexport WHERE id = :id LIMIT 1", + ['id' => $shopid] + ); + } else { + $einstellungen_json = $this->app->DB->Select( + "SELECT einstellungen_json FROM shopexport WHERE id = '$shopid' LIMIT 1" + ); + } $einstellungen = []; if (!empty($einstellungen_json)) { $einstellungen = json_decode($einstellungen_json, true) ?: []; @@ -954,10 +963,17 @@ public function persistLastImportTimestamp($isoUtcDate) $einstellungen['felder'] = []; } $einstellungen['felder']['letzter_import_timestamp'] = $isoUtcDate; - $this->app->DatabaseService->execute( - "UPDATE shopexport SET einstellungen_json = :json WHERE id = :id", - ['json' => json_encode($einstellungen), 'id' => (int)$this->shopid] - ); + $jsonEncoded = $this->app->DB->real_escape_string(json_encode($einstellungen)); + if (!empty($this->app->DatabaseService)) { + $this->app->DatabaseService->execute( + "UPDATE shopexport SET einstellungen_json = :json WHERE id = :id", + ['json' => json_encode($einstellungen), 'id' => $shopid] + ); + } else { + $this->app->DB->Update( + "UPDATE shopexport SET einstellungen_json = '$jsonEncoded' WHERE id = '$shopid'" + ); + } $this->lastImportTimestamp = $isoUtcDate; } @@ -2135,7 +2151,10 @@ protected function buildUrlQuery($url, $parameters = []) protected function authenticate($url, $method, $parameters = []) { // Setup authentication. - if ($this->isSsl()) { + // When query_string_auth is set, always use Basic Auth (consumer key/secret + // as query parameters), regardless of SSL. This allows HTTP-only test + // setups where the WooCommerce mu-plugin already whitelists the endpoint. + if ($this->isSsl() || $this->options->isQueryStringAuth()) { $basicAuth = new WCBasicAuth( $this->ch, $this->consumerKey, From 0d33e0ef2b9b53a066750e7dd172f75e9a669598 Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 17:27:52 +0200 Subject: [PATCH 08/18] revert(woocommerce): restore SSL-only gating for query-string basic auth WCHttpClient::authenticate() had isQueryStringAuth() smuggled into its SSL-gating during f12b09a4. That changed the auth scheme for existing HTTP-configured shops from OAuth 1.0a to basic-auth-over-query-string (since query_string_auth=true is set at client construction site). Restores the pre-f12b09a4 behaviour. The CLI-context fallback for persistLastImportTimestamp from f12b09a4 is kept. Co-Authored-By: Claude Sonnet 4.6 --- www/pages/shopimporter_woocommerce.php | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/www/pages/shopimporter_woocommerce.php b/www/pages/shopimporter_woocommerce.php index bfcdb0f10..f2406cfc7 100644 --- a/www/pages/shopimporter_woocommerce.php +++ b/www/pages/shopimporter_woocommerce.php @@ -2151,10 +2151,7 @@ protected function buildUrlQuery($url, $parameters = []) protected function authenticate($url, $method, $parameters = []) { // Setup authentication. - // When query_string_auth is set, always use Basic Auth (consumer key/secret - // as query parameters), regardless of SSL. This allows HTTP-only test - // setups where the WooCommerce mu-plugin already whitelists the endpoint. - if ($this->isSsl() || $this->options->isQueryStringAuth()) { + if ($this->isSsl()) { $basicAuth = new WCBasicAuth( $this->ch, $this->consumerKey, From dfe8fbf140df42422f7ae19990aba0d2424ba27d Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 17:29:11 +0200 Subject: [PATCH 09/18] fix(woocommerce): return single order per ImportGetAuftrag call The caller in shopimport.php uses $result[0] per iteration of a for loop capped by ImportGetAuftraegeAnzahl() and maxmanuell. Returning 500 orders per call therefore silently dropped 499 of them while advancing the server-side after-cursor past them. Restores the historical 1-order contract; the after-filter still replaces the legacy 800-id include hack, and per-order persist gives us resume-after-crash semantics with at most one order lost per crash (consistent with pre-#262 behaviour). MAX_PAGES_PER_RUN and ORDERS_PER_PAGE constants are removed; the caller loop (bounded by maxmanuell, default 100) now owns the batch size. Follow-up to review on #262. Co-Authored-By: Claude Sonnet 4.6 --- www/pages/shopimporter_woocommerce.php | 104 ++++++++++--------------- 1 file changed, 41 insertions(+), 63 deletions(-) diff --git a/www/pages/shopimporter_woocommerce.php b/www/pages/shopimporter_woocommerce.php index f2406cfc7..b90f64062 100644 --- a/www/pages/shopimporter_woocommerce.php +++ b/www/pages/shopimporter_woocommerce.php @@ -69,9 +69,6 @@ class Shopimporter_Woocommerce extends ShopimporterBase /** @var bool $lastImportTimestampIsFallback True when lastImportTimestamp was computed as 30-day fallback */ public $lastImportTimestampIsFallback = false; - const MAX_PAGES_PER_RUN = 5; - const ORDERS_PER_PAGE = 100; - public function __construct($app, $intern = false) { $this->app = $app; @@ -124,88 +121,69 @@ public function ImportGetAuftraegeAnzahl() } /** - * Queries the WooCommerce API for pending orders since the last import - * timestamp and returns them as a Xentral-formatted array. + * Queries the WooCommerce API for the oldest pending order since the last + * import timestamp and returns it as a Xentral-formatted array with at most + * one element. The caller (shopimport.php::RemoteGetAuftrag loop) expects + * $result[0] per iteration; this contract must be maintained. * - * Uses the WC v3 after= filter and paginates up to MAX_PAGES_PER_RUN - * pages (500 orders max per run). Progress timestamp is persisted after - * each processed order so aborted runs resume cleanly. + * The after-filter advances per order so each caller-iteration fetches the + * next order. A crash between RemoteGetAuftrag() and the shopimport_auftraege + * INSERT loses at most this one order (consistent with pre-#262 behaviour). * - * @return array|null + * @return array Array with at most one order entry, or empty array if none. */ public function ImportGetAuftrag() { $data = $this->CatchRemoteCommand('data'); + $this->getKonfig($data['shopid'] ?? null, $data); - // Transition: if timestamp is still a fallback and a legacy ab_nummer - // cursor is present, resolve it to a timestamp once. + // Transition: ab_nummer -> timestamp once, if we still have the fallback. if ($this->lastImportTimestampIsFallback && !empty($data['ab_nummer'])) { $resolved = $this->resolveAbNummerToTimestamp((int) $data['ab_nummer']); if ($resolved !== null) { $this->persistLastImportTimestamp($resolved); } - // lastImportTimestampIsFallback stays true only if resolution failed; - // persistLastImportTimestamp() updated $this->lastImportTimestamp already. } $configuredStatuses = array_map('trim', explode(';', (string) $this->statusPending)); - $runStartTimestamp = $this->lastImportTimestamp; - $page = 1; - $result = []; - - while ($page <= self::MAX_PAGES_PER_RUN) { - try { - $pageOrders = $this->client->get('orders', [ - 'status' => $configuredStatuses, - 'after' => $runStartTimestamp, - 'per_page' => self::ORDERS_PER_PAGE, - 'page' => $page, - 'orderby' => 'date', - 'order' => 'asc', - ]); - } catch (Exception $e) { - $this->logger->warning('WooCommerce ImportGetAuftrag page=' . $page . ': ' . $e->getMessage()); - break; - } - - if (empty($pageOrders)) { - break; - } - foreach ($pageOrders as $wcOrder) { - if (is_null($wcOrder)) { - continue; - } - - $order = $this->parseOrder($wcOrder); - - $result[] = [ - 'id' => $order['auftrag'], - 'sessionid' => '', - 'logdatei' => '', - 'warenkorb' => base64_encode(serialize($order)), - ]; + try { + $pageOrders = $this->client->get('orders', [ + 'status' => $configuredStatuses, + 'after' => $this->lastImportTimestamp, + 'per_page' => 1, + 'page' => 1, + 'orderby' => 'date', + 'order' => 'asc', + ]); + } catch (Exception $e) { + $this->logger->warning('WooCommerce ImportGetAuftrag: ' . $e->getMessage()); + return []; + } - // Persist progress after every order so partial runs resume cleanly. - if (!empty($wcOrder->date_created_gmt)) { - $this->persistLastImportTimestamp((string) $wcOrder->date_created_gmt); - } - } + if (empty($pageOrders)) { + return []; + } - $wcResponse = $this->client->getLastResponse(); - if ($wcResponse === null) { - break; - } + $wcOrder = $pageOrders[0] ?? null; + if ($wcOrder === null) { + return []; + } - $totalPages = (int) $wcResponse->getHeader('x-wp-totalpages'); - if ($totalPages < 1 || $page >= $totalPages) { - break; - } + $order = $this->parseOrder($wcOrder); - $page++; + // Persist progress for this order so the next Caller-iteration gets the + // next order via the after-filter. + if (!empty($wcOrder->date_created_gmt)) { + $this->persistLastImportTimestamp((string) $wcOrder->date_created_gmt); } - return !empty($result) ? $result : null; + return [[ + 'id' => $order['auftrag'], + 'sessionid' => '', + 'logdatei' => '', + 'warenkorb' => base64_encode(serialize($order)), + ]]; } /** From 1a07e3b14f9402be2d90ffe1d3a2b8429bec6f8f Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 17:30:33 +0200 Subject: [PATCH 10/18] docs: update pagination plan to reflect single-order contract After the external review of #262 highlighted that shopimport.php expects $result[0] per RemoteGetAuftrag iteration, the internal pagination loop was dropped. Plan now reflects: single-order per call, caller loop bounded by maxmanuell, per-order progress persist. Co-Authored-By: Claude Sonnet 4.6 --- docs/plans/woocommerce-pagination-fix.md | 91 +++++++++++------------- 1 file changed, 43 insertions(+), 48 deletions(-) diff --git a/docs/plans/woocommerce-pagination-fix.md b/docs/plans/woocommerce-pagination-fix.md index ce643be46..1ecd9a5c6 100644 --- a/docs/plans/woocommerce-pagination-fix.md +++ b/docs/plans/woocommerce-pagination-fix.md @@ -37,11 +37,10 @@ und verlustfrei, ohne neue Plugin-Abhängigkeiten und ohne Wechsel der API-Versi | Parameter | Wert | Begruendung | |---|---|---| -| `MAX_PAGES_PER_RUN` | **5** | Max. 500 Bestellungen pro Cron-Lauf. Bei groesseren Rueckstaenden arbeiten nachfolgende Laeufe die Restmenge ab. Verhindert Cron-Timeouts. | -| `ORDERS_PER_PAGE` | **100** | WC-v3-Default-Maximum. Minimiert Roundtrips. | | Erstlauf-Fallback | **30 Tage** | Sinnvoller Mittelweg: holt historische Bestellungen, aber nicht unbegrenzt. UI-Override in Folge-PR moeglich. | | Persistenz-Feld | `shopexport.einstellungen_json` → `felder.letzter_import_timestamp` | Bestehende Struktur nutzen, keine DB-Migration. | | Timestamp-Format | ISO-8601 `Y-m-d\TH:i:s` (UTC) | Direkt an `after=` durchreichbar. | +| Caller-Cap (pre-existing) | `shopexport.maxmanuell`, default 100 | Der Batch-Cap liegt beim shopimport.php-Caller, nicht im Importer. | ## 4. Datei- und Funktions-Landkarte @@ -50,7 +49,7 @@ Zielmodul: `www/pages/shopimporter_woocommerce.php` | Funktion | Zeilen | Betroffen? | Aenderung | |---|---|---|---| | `ImportGetAuftraegeAnzahl` | ~80–135 | ja | `include`-Hack raus, `after`-Filter rein, nur noch Count via `X-WP-Total` | -| `ImportGetAuftrag` | ~138–220 | ja | Seiten-Schleife, `after`-Filter, Timestamp-Update am Ende | +| `ImportGetAuftrag` | ~121–185 | ja | Single-Order-Query mit after-Filter; Caller-Loop in shopimport.php iteriert | | `parseOrder` | ~222–305 | nein | unveraendert (liefert weiterhin pro Order) | | `CatchRemoteCommand('data')` | mehrfach | mittelbar | stellt `letzter_import_timestamp` aus `einstellungen_json` bereit | | `getKonfig` | ~802–837 | nein | nicht anfassen (separates Issue #224) | @@ -139,58 +138,53 @@ SELECT einstellungen_json FROM shopexport WHERE id = ### Schritt 5 — Refactor `ImportGetAuftrag` (Import-Funktion) -**Alt (Zeile 138–220):** -- Query: `orders?status=…&include=<800 IDs>&per_page=20&orderby=id&order=asc`. -- Iteriert einmalig ueber Response, ruft `parseOrder` pro Eintrag. +**Alt:** Query mit `include`-Liste, Iteration ueber bis zu 20 Orders, kein Cursor. -**Neu — Pseudocode-Anker (keine OpenXE-Implementierung, nur Logik):** +**Neu — Single-Order-Pseudocode:** ``` -const MAX_PAGES_PER_RUN = 5 -const ORDERS_PER_PAGE = 100 - -lastTs = $this->lastImportTimestamp -orders = [] -page = 1 - -while page <= MAX_PAGES_PER_RUN: - response = client.get('orders', { - status[]: $configuredStatuses, - after: lastTs, - per_page: ORDERS_PER_PAGE, - page: page, - orderby: 'date', - order: 'asc', - }) - - if response is empty: - break - - for order in response: - parsed = parseOrder(order) - orders.push(parsed) - # Progress-Timestamp: nach jeder Order fortschreiben - this.persistLastImportTimestamp(order.date_created_gmt) - - totalPages = (int) response.getHeader('x-wp-totalpages') - if page >= totalPages: - break - - page++ - -return orders +response = client.get('orders', { + status[]: configuredStatuses, + after: this.lastImportTimestamp, + per_page: 1, + page: 1, + orderby: 'date', + order: 'asc', +}) + +if response is empty: + return [] + +wcOrder = response[0] +order = parseOrder(wcOrder) + +# Cursor auf diese Order setzen, damit naechste Caller-Iteration +# die naechste Order holt. +this.persistLastImportTimestamp(wcOrder.date_created_gmt) + +return [{ id: order.auftrag, sessionid: '', logdatei: '', warenkorb: ... }] ``` **Wichtig:** - `orderby=date` + `order=asc` garantiert, dass die **aelteste** neue Order zuerst kommt. Dadurch kann der Progress-Timestamp monoton wachsen. - `date_created_gmt` als Referenz (nicht `date_created` — Zeitzonen-Fallen vermeiden). -- `MAX_PAGES_PER_RUN` ist eine Konstante am Klassenanfang, nicht per-Config (kann in - Folge-PR konfigurierbar werden). +- Volume-Handling liegt beim Caller (shopimport.php), nicht im Importer. **Akzeptanzkriterium:** -- Bei 250 neuen Orders ueber einen Lauf: alle 250 importiert. -- Bei 750 neuen Orders: Lauf 1 importiert 500, Lauf 2 importiert 250. +- Pro Call: exakt 1 Order zurueck (oder leer). +- Bei >100 neuen Orders: Caller-Schleife (maxmanuell-gekappt) iteriert bis Ende. + +### Schritt 5a — Caller-Kontrakt + +Der Caller in shopimport.php:1304-1306 ruft pro Order-Import-Iteration: +- `ImportGetAuftraegeAnzahl()` einmal fuer Count (mit maxmanuell-Cap) +- `ImportGetAuftrag()` in for-Schleife, verarbeitet `$result[0]` + +Der Importer muss diesen Kontrakt einhalten: pro Call max. 1 Order. +Der after-Filter sorgt dafuer, dass jede Iteration die naechste Order +holt. Ein Crash zwischen `RemoteGetAuftrag()` und `shopimport_auftraege`- +Insert verliert max. diese eine Order. ### Schritt 6 — Transitions-Kompatibilitaet `ab_nummer` @@ -260,11 +254,11 @@ Manuelle Integrationstests mit seeded Test-Orders. | T2 | Frischinstall, 10 Orders <30 Tage alt | `letzter_import_timestamp` leer | Lauf starten | 10 Orders importiert, Timestamp = neueste Order | | T3 | Frischinstall, 10 Orders >30 Tage alt | `letzter_import_timestamp` leer | Lauf starten | 0 Orders (Fallback-Schwelle), Timestamp leer | | T4 | Standardlauf, 30 neue Orders | Timestamp gesetzt | Lauf starten | 30 Orders, Timestamp fortgeschrieben | -| T5 | Spike, 150 neue Orders | Timestamp gesetzt | Lauf starten | 150 Orders (1 Lauf, 2 Seiten), Timestamp fortgeschrieben | -| T6 | Rueckstand, 750 neue Orders | Timestamp gesetzt | Lauf 1 starten | 500 Orders importiert, `MAX_PAGES_PER_RUN`-Cap greift | -| T6b | Rueckstand Teil 2 | nach T6 | Lauf 2 starten | restliche 250 Orders, Timestamp final | +| T5 | Spike, 150 neue Orders | Timestamp gesetzt | Lauf starten | 150 Orders, Caller-Schleife durch maxmanuell auf 100 gekappt; Folgelauf holt Rest | +| T6 | Rueckstand, >100 neue Orders | Timestamp gesetzt | Lauf 1 starten | 100 Orders (maxmanuell-Cap), Cursor fortgeschrieben | +| T6b | Rueckstand Teil 2 | nach T6 | Lauf 2 starten | restliche Orders bis maxmanuell-Cap, Timestamp final | | T7 | Transition, Shop mit `ab_nummer`=12345, Timestamp leer | `ab_nummer=12345` | Lauf starten | `ab_nummer` in Timestamp uebersetzt, keine Duplikate | -| T8 | Abbruch mitten im Lauf | 50 von 100 verarbeitet, dann Exception | neuer Lauf | wieder bei Order 51 starten, nicht doppelt importiert | +| T8 | Abbruch mitten im Lauf | Exception nach Fetch, vor INSERT | neuer Lauf | bei Abbruch zwischen Fetch und Insert max. 1 Order verloren; ab naechster Order fortgesetzt | | T9 | Idempotenz | T4 erfolgreich gelaufen | T4 nochmal starten | 0 neue Orders, keine Duplikate | | T10 | URL-Laenge (Regression) | 800 alte Orders existieren | Lauf starten | URL bleibt kurz (<2 KB), keine `include`-Liste mehr | @@ -304,6 +298,7 @@ Manuelle Integrationstests mit seeded Test-Orders. | `orderby=date` Ordering nicht deterministisch bei gleichem Timestamp | niedrig | niedrig | Zusaetzlich `orderby=date&order=asc` und WC liefert stabile Sekundaersortierung nach ID | | Andere Shopimporter erben Basisklasse-Struktur | niedrig | niedrig | Nur `shopimporter_woocommerce.php` anfassen, `ShopimporterBase` nicht anruehren | | URL-Laengenlimits der WAF beim `status[]`-Array | sehr niedrig | niedrig | Typisch 2-3 Status-Werte, URL bleibt kurz | +| Caller-Kontrakt-Abweichung (1 vs. n Orders) | niedrig | hoch | Per Design Single-Order-Return; Caller-Loop fuer Volume-Handling | ## 9. Definition of Done From d32c4c0a34dc2ec57b85117f42596c302a0e4278 Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 17:36:40 +0200 Subject: [PATCH 11/18] fix(woocommerce): return null (not empty array) from ImportGetAuftrag when nothing to import MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit shopimport.php:1308 gates on is_array($result), which happily accepts [] and then crashes trying to dereference $result[0]['id']. The pre-#262 behaviour returned null on empty — restore that. Spotted by review of 7af9edb7. --- www/pages/shopimporter_woocommerce.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/www/pages/shopimporter_woocommerce.php b/www/pages/shopimporter_woocommerce.php index b90f64062..8412e65ea 100644 --- a/www/pages/shopimporter_woocommerce.php +++ b/www/pages/shopimporter_woocommerce.php @@ -158,16 +158,16 @@ public function ImportGetAuftrag() ]); } catch (Exception $e) { $this->logger->warning('WooCommerce ImportGetAuftrag: ' . $e->getMessage()); - return []; + return null; } if (empty($pageOrders)) { - return []; + return null; } $wcOrder = $pageOrders[0] ?? null; if ($wcOrder === null) { - return []; + return null; } $order = $this->parseOrder($wcOrder); From 709991647c798d0b55391d32a859c60d453fd4cf Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 17:48:04 +0200 Subject: [PATCH 12/18] fix(woocommerce): remove broken getKonfig() re-init in ImportGetAuftrag The defensive getKonfig($data['shopid'] ?? null, $data) call inside ImportGetAuftrag() is actively harmful: CatchRemoteCommand('data') returns the getauftrag-payload, which does not carry a shopid (see class.remote.php:194/241). The re-init therefore clears $this->shopid and rebuilds the WCClient from empty preferences. RemoteCommand() already initialises the importer with the real shop id before dispatch (class.remote.php:2685), so the duplicate call is both redundant and broken. Spotted by re-review of #262. --- www/pages/shopimporter_woocommerce.php | 1 - 1 file changed, 1 deletion(-) diff --git a/www/pages/shopimporter_woocommerce.php b/www/pages/shopimporter_woocommerce.php index 8412e65ea..5904f4729 100644 --- a/www/pages/shopimporter_woocommerce.php +++ b/www/pages/shopimporter_woocommerce.php @@ -135,7 +135,6 @@ public function ImportGetAuftraegeAnzahl() public function ImportGetAuftrag() { $data = $this->CatchRemoteCommand('data'); - $this->getKonfig($data['shopid'] ?? null, $data); // Transition: ab_nummer -> timestamp once, if we still have the fallback. if ($this->lastImportTimestampIsFallback && !empty($data['ab_nummer'])) { From 0af53efac6237812499272418cf22b272ca48420 Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 17:48:48 +0200 Subject: [PATCH 13/18] fix(woocommerce): run ab_nummer migration in the count path, too shopimport.php calls RemoteGetAuftraegeAnzahl() before RemoteGetAuftrag() in its main flow. If the stored cursor is still the 30-day fallback and all pending orders are older than 30 days, the count returns 0 and ImportGetAuftrag() never runs, so the one-shot ab_nummer -> timestamp migration never fires and the shop stays on the fallback forever. Extract the migration into a private helper and invoke it from both count and fetch paths. Idempotent via lastImportTimestampIsFallback. --- www/pages/shopimporter_woocommerce.php | 31 ++++++++++++++++++++------ 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/www/pages/shopimporter_woocommerce.php b/www/pages/shopimporter_woocommerce.php index 5904f4729..3f9d2f78e 100644 --- a/www/pages/shopimporter_woocommerce.php +++ b/www/pages/shopimporter_woocommerce.php @@ -92,6 +92,8 @@ public function ImportList() */ public function ImportGetAuftraegeAnzahl() { + $this->migrateAbNummerIfNeeded(); + $configuredStatuses = array_map('trim', explode(';', (string) $this->statusPending)); try { @@ -136,13 +138,7 @@ public function ImportGetAuftrag() { $data = $this->CatchRemoteCommand('data'); - // Transition: ab_nummer -> timestamp once, if we still have the fallback. - if ($this->lastImportTimestampIsFallback && !empty($data['ab_nummer'])) { - $resolved = $this->resolveAbNummerToTimestamp((int) $data['ab_nummer']); - if ($resolved !== null) { - $this->persistLastImportTimestamp($resolved); - } - } + $this->migrateAbNummerIfNeeded(); $configuredStatuses = array_map('trim', explode(';', (string) $this->statusPending)); @@ -214,6 +210,27 @@ private function resolveAbNummerToTimestamp($abNummer) return gmdate('Y-m-d\TH:i:s', $ts - 1); } + /** + * Runs the one-shot legacy ab_nummer -> timestamp migration when the stored + * cursor is still the 30-day fallback and the caller passes an ab_nummer. + * Idempotent: once migrated, lastImportTimestampIsFallback flips to false + * and subsequent calls become no-ops. + */ + private function migrateAbNummerIfNeeded() + { + if (!$this->lastImportTimestampIsFallback) { + return; + } + $data = $this->CatchRemoteCommand('data'); + if (empty($data['ab_nummer'])) { + return; + } + $resolved = $this->resolveAbNummerToTimestamp((int) $data['ab_nummer']); + if ($resolved !== null) { + $this->persistLastImportTimestamp($resolved); + } + } + // This function searches the wcOrder for the specified WC Meta key // and returns it if found, null otherise public function get_wc_meta($wcOrder, $meta_key) From 78545a4ad475978b9c28ca04297940fe17f37588 Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 17:50:13 +0200 Subject: [PATCH 14/18] fix(woocommerce): tuple cursor (ts, id) to survive same-second orders The after-filter is strictly-greater-than, so orders sharing an identical date_created_gmt with the last processed order were silently dropped. Move to a tuple cursor: persist both timestamp and order id, query with after= plus exclude=[last_id]. Orders with the same GMT second now reach the caller in subsequent iterations without duplicating the already-processed one. Schema is additive (new felder.letzter_import_order_id key in shopexport.einstellungen_json). Persistence helper becomes persistLastImportCursor; the single-argument persistLastImportTimestamp remains as a wrapper so the ab_nummer migration path keeps working without a second rewrite. --- www/pages/shopimporter_woocommerce.php | 77 +++++++++++++++++++------- 1 file changed, 58 insertions(+), 19 deletions(-) diff --git a/www/pages/shopimporter_woocommerce.php b/www/pages/shopimporter_woocommerce.php index 3f9d2f78e..dc2c6648f 100644 --- a/www/pages/shopimporter_woocommerce.php +++ b/www/pages/shopimporter_woocommerce.php @@ -69,6 +69,9 @@ class Shopimporter_Woocommerce extends ShopimporterBase /** @var bool $lastImportTimestampIsFallback True when lastImportTimestamp was computed as 30-day fallback */ public $lastImportTimestampIsFallback = false; + /** @var int|null $lastImportOrderId WooCommerce order ID of the last successfully imported order */ + public $lastImportOrderId = null; + public function __construct($app, $intern = false) { $this->app = $app; @@ -96,12 +99,18 @@ public function ImportGetAuftraegeAnzahl() $configuredStatuses = array_map('trim', explode(';', (string) $this->statusPending)); + $afterTs = gmdate('Y-m-d\TH:i:s', max(0, strtotime($this->lastImportTimestamp) - 1)); + $queryArgs = [ + 'status' => $configuredStatuses, + 'after' => $afterTs, + 'per_page' => 1, + ]; + if (!empty($this->lastImportOrderId)) { + $queryArgs['exclude'] = [(int) $this->lastImportOrderId]; + } + try { - $this->client->get('orders', [ - 'status' => $configuredStatuses, - 'after' => $this->lastImportTimestamp, - 'per_page' => 1, - ]); + $this->client->get('orders', $queryArgs); } catch (Exception $e) { $this->logger->warning('WooCommerce ImportGetAuftraegeAnzahl: API request failed: ' . $e->getMessage()); return 0; @@ -142,15 +151,21 @@ public function ImportGetAuftrag() $configuredStatuses = array_map('trim', explode(';', (string) $this->statusPending)); + $afterTs = gmdate('Y-m-d\TH:i:s', max(0, strtotime($this->lastImportTimestamp) - 1)); + $queryArgs = [ + 'status' => $configuredStatuses, + 'after' => $afterTs, + 'per_page' => 1, + 'page' => 1, + 'orderby' => 'date', + 'order' => 'asc', + ]; + if (!empty($this->lastImportOrderId)) { + $queryArgs['exclude'] = [(int) $this->lastImportOrderId]; + } + try { - $pageOrders = $this->client->get('orders', [ - 'status' => $configuredStatuses, - 'after' => $this->lastImportTimestamp, - 'per_page' => 1, - 'page' => 1, - 'orderby' => 'date', - 'order' => 'asc', - ]); + $pageOrders = $this->client->get('orders', $queryArgs); } catch (Exception $e) { $this->logger->warning('WooCommerce ImportGetAuftrag: ' . $e->getMessage()); return null; @@ -167,10 +182,11 @@ public function ImportGetAuftrag() $order = $this->parseOrder($wcOrder); - // Persist progress for this order so the next Caller-iteration gets the - // next order via the after-filter. - if (!empty($wcOrder->date_created_gmt)) { - $this->persistLastImportTimestamp((string) $wcOrder->date_created_gmt); + // Persist tuple cursor so the next Caller-iteration advances past this order. + // Using ts-1s in the query means same-second peers are still fetched, while + // the exclude=[id] parameter prevents re-delivering this exact order. + if (!empty($wcOrder->date_created_gmt) && !empty($wcOrder->id)) { + $this->persistLastImportCursor((string) $wcOrder->date_created_gmt, (int) $wcOrder->id); } return [[ @@ -925,16 +941,33 @@ public function getKonfig($shopid, $data) $this->lastImportTimestampIsFallback = true; } + $this->lastImportOrderId = isset($preferences['felder']['letzter_import_order_id']) + ? (int) $preferences['felder']['letzter_import_order_id'] + : null; + } /** - * Persists the last successful import timestamp to shopexport.einstellungen_json. - * Does a read-modify-write to preserve all other fields. + * Backwards-compatible wrapper: persists timestamp only (order id cleared). + * Use persistLastImportCursor() when both timestamp and order id are available. * * @param string $isoUtcDate ISO-8601 UTC timestamp, e.g. '2026-04-20T12:34:56' * @return void */ public function persistLastImportTimestamp($isoUtcDate) + { + $this->persistLastImportCursor($isoUtcDate, null); + } + + /** + * Persists the tuple cursor (timestamp + order id) to shopexport.einstellungen_json. + * Does a read-modify-write to preserve all other fields. + * + * @param string $isoUtcDate ISO-8601 UTC timestamp, e.g. '2026-04-20T12:34:56' + * @param int|null $orderId WooCommerce order ID, or null to clear + * @return void + */ + public function persistLastImportCursor($isoUtcDate, $orderId = null) { $shopid = (int)$this->shopid; // Prefer DatabaseService when available (web context), fall back to DB @@ -957,6 +990,11 @@ public function persistLastImportTimestamp($isoUtcDate) $einstellungen['felder'] = []; } $einstellungen['felder']['letzter_import_timestamp'] = $isoUtcDate; + if ($orderId !== null) { + $einstellungen['felder']['letzter_import_order_id'] = (int) $orderId; + } else { + unset($einstellungen['felder']['letzter_import_order_id']); + } $jsonEncoded = $this->app->DB->real_escape_string(json_encode($einstellungen)); if (!empty($this->app->DatabaseService)) { $this->app->DatabaseService->execute( @@ -969,6 +1007,7 @@ public function persistLastImportTimestamp($isoUtcDate) ); } $this->lastImportTimestamp = $isoUtcDate; + $this->lastImportOrderId = $orderId !== null ? (int) $orderId : null; } /** From f3055b5298ef9b32cb81806dabfc1eb96d709ac7 Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 17:51:03 +0200 Subject: [PATCH 15/18] docs: extend pagination plan to cover tuple cursor and count-path migration Reflects re-review findings: tuple cursor (ts, id) for same-second order resilience, migration helper called from both count and fetch paths, scope list updated to match current single-order design. --- docs/plans/woocommerce-pagination-fix.md | 29 ++++++++++++++---------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/docs/plans/woocommerce-pagination-fix.md b/docs/plans/woocommerce-pagination-fix.md index 1ecd9a5c6..f367255eb 100644 --- a/docs/plans/woocommerce-pagination-fix.md +++ b/docs/plans/woocommerce-pagination-fix.md @@ -18,7 +18,7 @@ und verlustfrei, ohne neue Plugin-Abhängigkeiten und ohne Wechsel der API-Versi ### IN-Scope -- Pagination-Loop beim Bestellabruf (`X-WP-TotalPages`-Header auswerten) +- Single-Order-Abruf mit `after`-Filter + `exclude`-Tupel-Cursor; Caller-Loop iteriert. - `after`-Filter (ISO-8601) statt 800er-`include`-Hack - Persistenz "letzter erfolgreicher Import-Timestamp" pro Shop - Fallback-Logik für Erstlauf (kein Timestamp vorhanden) @@ -41,6 +41,7 @@ und verlustfrei, ohne neue Plugin-Abhängigkeiten und ohne Wechsel der API-Versi | Persistenz-Feld | `shopexport.einstellungen_json` → `felder.letzter_import_timestamp` | Bestehende Struktur nutzen, keine DB-Migration. | | Timestamp-Format | ISO-8601 `Y-m-d\TH:i:s` (UTC) | Direkt an `after=` durchreichbar. | | Caller-Cap (pre-existing) | `shopexport.maxmanuell`, default 100 | Der Batch-Cap liegt beim shopimport.php-Caller, nicht im Importer. | +| Cursor-Format | Tupel `(letzter_import_timestamp, letzter_import_order_id)` | Loest Same-Second-Kollisionen via after=ts-1 + exclude=[id]. | ## 4. Datei- und Funktions-Landkarte @@ -143,14 +144,9 @@ SELECT einstellungen_json FROM shopexport WHERE id = **Neu — Single-Order-Pseudocode:** ``` -response = client.get('orders', { - status[]: configuredStatuses, - after: this.lastImportTimestamp, - per_page: 1, - page: 1, - orderby: 'date', - order: 'asc', -}) +afterTs = ts-1s (falls ts gesetzt) +query = orders?after=&per_page=1&orderby=date&order=asc + (+ exclude=[last_id] wenn last_id bekannt) if response is empty: return [] @@ -158,9 +154,7 @@ if response is empty: wcOrder = response[0] order = parseOrder(wcOrder) -# Cursor auf diese Order setzen, damit naechste Caller-Iteration -# die naechste Order holt. -this.persistLastImportTimestamp(wcOrder.date_created_gmt) +persistLastImportCursor(order.date_created_gmt, order.id) return [{ id: order.auftrag, sessionid: '', logdatei: '', warenkorb: ... }] ``` @@ -186,6 +180,16 @@ Der after-Filter sorgt dafuer, dass jede Iteration die naechste Order holt. Ein Crash zwischen `RemoteGetAuftrag()` und `shopimport_auftraege`- Insert verliert max. diese eine Order. +### Schritt 5b — Migration-Helper + +Um die ab_nummer → timestamp Migration auch im Count-Pfad auszufuehren +(shopimport.php ruft ImportGetAuftraegeAnzahl() BEFORE ImportGetAuftrag()): + +- Extraktion in eine private Methode `migrateAbNummerIfNeeded()`. +- Aufruf am Anfang von beiden ImportGetAuftraegeAnzahl() und ImportGetAuftrag(). +- Idempotent durch `$lastImportTimestampIsFallback`-Check — nach einmaliger Migration + keine weiteren Reads. + ### Schritt 6 — Transitions-Kompatibilitaet `ab_nummer` **Ist-Zustand:** `CatchRemoteCommand('data')` liefert u.a. `ab_nummer` — die naechste @@ -299,6 +303,7 @@ Manuelle Integrationstests mit seeded Test-Orders. | Andere Shopimporter erben Basisklasse-Struktur | niedrig | niedrig | Nur `shopimporter_woocommerce.php` anfassen, `ShopimporterBase` nicht anruehren | | URL-Laengenlimits der WAF beim `status[]`-Array | sehr niedrig | niedrig | Typisch 2-3 Status-Werte, URL bleibt kurz | | Caller-Kontrakt-Abweichung (1 vs. n Orders) | niedrig | hoch | Per Design Single-Order-Return; Caller-Loop fuer Volume-Handling | +| Timestamp-Kollision bei identischem date_created_gmt | niedrig | mittel | Adressiert durch Tupel-Cursor (ts, id) + exclude-Parameter | ## 9. Definition of Done From dbddc49f5c8c3bace59e26bf47220b51ef820416 Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 18:54:45 +0200 Subject: [PATCH 16/18] fix(woocommerce): accumulate same-second order ids in cursor, gate -1s offset Bei identischem date_created_gmt mehrerer Orders hielt exclude nur die zuletzt importierte ID. Nach zwei Orders im selben Bucket wurde die erste wieder sichtbar und Count- wie Fetch-Pfad liefen in eine Endlosschleife. Cursor persistiert jetzt die komplette Liste aller IDs innerhalb des aktuellen Sekunden-Buckets (felder.letzter_import_order_ids als JSON- Array). Bei Bucket-Wechsel wird die Liste zurueckgesetzt; bei gleichem Bucket wird die neue ID angehaengt. Gleichzeitig wird die -1s-Korrektur am Query gated: nur wenn mindestens eine exclude-ID bekannt ist, wird der after-Filter um 1 Sekunde nach hinten verschoben. Dadurch entfaellt die Doppel-Subtraktion nach der ab_nummer-Migration (resolveAbNummerToTimestamp schon -1s, Query war nochmal -1s -> 2s zurueck in der Vergangenheit). Der Erstlauf nach Migration liefert jetzt exakt die ab_nummer-Order als Startpunkt. Co-Authored-By: Claude Sonnet 4.6 --- www/pages/shopimporter_woocommerce.php | 117 +++++++++++++++++-------- 1 file changed, 79 insertions(+), 38 deletions(-) diff --git a/www/pages/shopimporter_woocommerce.php b/www/pages/shopimporter_woocommerce.php index dc2c6648f..9eccc455a 100644 --- a/www/pages/shopimporter_woocommerce.php +++ b/www/pages/shopimporter_woocommerce.php @@ -69,8 +69,8 @@ class Shopimporter_Woocommerce extends ShopimporterBase /** @var bool $lastImportTimestampIsFallback True when lastImportTimestamp was computed as 30-day fallback */ public $lastImportTimestampIsFallback = false; - /** @var int|null $lastImportOrderId WooCommerce order ID of the last successfully imported order */ - public $lastImportOrderId = null; + /** @var int[] $lastImportOrderIds WooCommerce order IDs within the current timestamp bucket */ + public $lastImportOrderIds = []; public function __construct($app, $intern = false) { @@ -99,14 +99,20 @@ public function ImportGetAuftraegeAnzahl() $configuredStatuses = array_map('trim', explode(';', (string) $this->statusPending)); - $afterTs = gmdate('Y-m-d\TH:i:s', max(0, strtotime($this->lastImportTimestamp) - 1)); - $queryArgs = [ - 'status' => $configuredStatuses, - 'after' => $afterTs, - 'per_page' => 1, - ]; - if (!empty($this->lastImportOrderId)) { - $queryArgs['exclude'] = [(int) $this->lastImportOrderId]; + if (!empty($this->lastImportOrderIds)) { + $afterTs = gmdate('Y-m-d\TH:i:s', max(0, strtotime($this->lastImportTimestamp) - 1)); + $queryArgs = [ + 'status' => $configuredStatuses, + 'after' => $afterTs, + 'per_page' => 1, + 'exclude' => array_values($this->lastImportOrderIds), + ]; + } else { + $queryArgs = [ + 'status' => $configuredStatuses, + 'after' => $this->lastImportTimestamp, + 'per_page' => 1, + ]; } try { @@ -151,17 +157,26 @@ public function ImportGetAuftrag() $configuredStatuses = array_map('trim', explode(';', (string) $this->statusPending)); - $afterTs = gmdate('Y-m-d\TH:i:s', max(0, strtotime($this->lastImportTimestamp) - 1)); - $queryArgs = [ - 'status' => $configuredStatuses, - 'after' => $afterTs, - 'per_page' => 1, - 'page' => 1, - 'orderby' => 'date', - 'order' => 'asc', - ]; - if (!empty($this->lastImportOrderId)) { - $queryArgs['exclude'] = [(int) $this->lastImportOrderId]; + if (!empty($this->lastImportOrderIds)) { + $afterTs = gmdate('Y-m-d\TH:i:s', max(0, strtotime($this->lastImportTimestamp) - 1)); + $queryArgs = [ + 'status' => $configuredStatuses, + 'after' => $afterTs, + 'per_page' => 1, + 'page' => 1, + 'orderby' => 'date', + 'order' => 'asc', + 'exclude' => array_values($this->lastImportOrderIds), + ]; + } else { + $queryArgs = [ + 'status' => $configuredStatuses, + 'after' => $this->lastImportTimestamp, + 'per_page' => 1, + 'page' => 1, + 'orderby' => 'date', + 'order' => 'asc', + ]; } try { @@ -941,9 +956,10 @@ public function getKonfig($shopid, $data) $this->lastImportTimestampIsFallback = true; } - $this->lastImportOrderId = isset($preferences['felder']['letzter_import_order_id']) - ? (int) $preferences['felder']['letzter_import_order_id'] - : null; + $storedIds = $preferences['felder']['letzter_import_order_ids'] ?? null; + $this->lastImportOrderIds = is_array($storedIds) + ? array_values(array_filter(array_map('intval', $storedIds))) + : []; } @@ -960,11 +976,17 @@ public function persistLastImportTimestamp($isoUtcDate) } /** - * Persists the tuple cursor (timestamp + order id) to shopexport.einstellungen_json. - * Does a read-modify-write to preserve all other fields. + * Persists the tuple cursor (timestamp + accumulated order-id bucket) to + * shopexport.einstellungen_json. Does a read-modify-write to preserve all + * other fields. + * + * Bucket logic: + * - $orderId === null → migration path; ids list is cleared. + * - same timestamp as stored → append $orderId to the ids list. + * - new timestamp → reset ids list to [$orderId]. * * @param string $isoUtcDate ISO-8601 UTC timestamp, e.g. '2026-04-20T12:34:56' - * @param int|null $orderId WooCommerce order ID, or null to clear + * @param int|null $orderId WooCommerce order ID, or null (migration path) * @return void */ public function persistLastImportCursor($isoUtcDate, $orderId = null) @@ -982,24 +1004,43 @@ public function persistLastImportCursor($isoUtcDate, $orderId = null) "SELECT einstellungen_json FROM shopexport WHERE id = '$shopid' LIMIT 1" ); } - $einstellungen = []; + $current = []; if (!empty($einstellungen_json)) { - $einstellungen = json_decode($einstellungen_json, true) ?: []; + $current = json_decode($einstellungen_json, true) ?: []; + } + if (!isset($current['felder']) || !is_array($current['felder'])) { + $current['felder'] = []; } - if (!isset($einstellungen['felder']) || !is_array($einstellungen['felder'])) { - $einstellungen['felder'] = []; + + $previousTs = $current['felder']['letzter_import_timestamp'] ?? null; + $previousIds = $current['felder']['letzter_import_order_ids'] ?? []; + if (!is_array($previousIds)) { + $previousIds = []; } - $einstellungen['felder']['letzter_import_timestamp'] = $isoUtcDate; - if ($orderId !== null) { - $einstellungen['felder']['letzter_import_order_id'] = (int) $orderId; + + // Determine ids list for the new state. + if ($orderId === null) { + // Migration path — timestamp without a concrete order-id anchor. + $newIds = []; + } elseif ($previousTs !== null && $isoUtcDate === $previousTs) { + // Same timestamp bucket — append id if not already present. + $newIds = $previousIds; + if (!in_array((int) $orderId, $newIds, true)) { + $newIds[] = (int) $orderId; + } } else { - unset($einstellungen['felder']['letzter_import_order_id']); + // New timestamp bucket — reset list to only this id. + $newIds = [(int) $orderId]; } - $jsonEncoded = $this->app->DB->real_escape_string(json_encode($einstellungen)); + + $current['felder']['letzter_import_timestamp'] = $isoUtcDate; + $current['felder']['letzter_import_order_ids'] = $newIds; + + $jsonEncoded = $this->app->DB->real_escape_string(json_encode($current)); if (!empty($this->app->DatabaseService)) { $this->app->DatabaseService->execute( "UPDATE shopexport SET einstellungen_json = :json WHERE id = :id", - ['json' => json_encode($einstellungen), 'id' => $shopid] + ['json' => json_encode($current), 'id' => $shopid] ); } else { $this->app->DB->Update( @@ -1007,7 +1048,7 @@ public function persistLastImportCursor($isoUtcDate, $orderId = null) ); } $this->lastImportTimestamp = $isoUtcDate; - $this->lastImportOrderId = $orderId !== null ? (int) $orderId : null; + $this->lastImportOrderIds = $newIds; } /** From e6dd616edf57d79bea246f13e101f873d782337e Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Mon, 20 Apr 2026 18:55:19 +0200 Subject: [PATCH 17/18] docs: document bucket-accumulating cursor and gated -1s offset Co-Authored-By: Claude Sonnet 4.6 --- docs/plans/woocommerce-pagination-fix.md | 25 +++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/docs/plans/woocommerce-pagination-fix.md b/docs/plans/woocommerce-pagination-fix.md index f367255eb..5d874b990 100644 --- a/docs/plans/woocommerce-pagination-fix.md +++ b/docs/plans/woocommerce-pagination-fix.md @@ -41,7 +41,7 @@ und verlustfrei, ohne neue Plugin-Abhängigkeiten und ohne Wechsel der API-Versi | Persistenz-Feld | `shopexport.einstellungen_json` → `felder.letzter_import_timestamp` | Bestehende Struktur nutzen, keine DB-Migration. | | Timestamp-Format | ISO-8601 `Y-m-d\TH:i:s` (UTC) | Direkt an `after=` durchreichbar. | | Caller-Cap (pre-existing) | `shopexport.maxmanuell`, default 100 | Der Batch-Cap liegt beim shopimport.php-Caller, nicht im Importer. | -| Cursor-Format | Tupel `(letzter_import_timestamp, letzter_import_order_id)` | Loest Same-Second-Kollisionen via after=ts-1 + exclude=[id]. | +| Cursor-Format | Tupel `(letzter_import_timestamp, letzter_import_order_ids: [int])` | IDs sammeln den aktuellen Sekunden-Bucket; -1s-Verschiebung greift nur mit mindestens einer ID. | ## 4. Datei- und Funktions-Landkarte @@ -144,9 +144,16 @@ SELECT einstellungen_json FROM shopexport WHERE id = **Neu — Single-Order-Pseudocode:** ``` -afterTs = ts-1s (falls ts gesetzt) -query = orders?after=&per_page=1&orderby=date&order=asc - (+ exclude=[last_id] wenn last_id bekannt) +# Fenster aufbauen +if last_order_ids is not empty: + after = last_ts - 1s + exclude = last_order_ids +else: + after = last_ts + exclude = (nicht gesetzt) + +# Fetch +order = client.get(orders, after=after, exclude=exclude, per_page=1, orderby=date, order=asc)[0] if response is empty: return [] @@ -154,6 +161,13 @@ if response is empty: wcOrder = response[0] order = parseOrder(wcOrder) +# Persist +if order.date_created_gmt == last_ts: + last_order_ids = last_order_ids + [order.id] +else: + last_ts = order.date_created_gmt + last_order_ids = [order.id] + persistLastImportCursor(order.date_created_gmt, order.id) return [{ id: order.auftrag, sessionid: '', logdatei: '', warenkorb: ... }] @@ -303,7 +317,8 @@ Manuelle Integrationstests mit seeded Test-Orders. | Andere Shopimporter erben Basisklasse-Struktur | niedrig | niedrig | Nur `shopimporter_woocommerce.php` anfassen, `ShopimporterBase` nicht anruehren | | URL-Laengenlimits der WAF beim `status[]`-Array | sehr niedrig | niedrig | Typisch 2-3 Status-Werte, URL bleibt kurz | | Caller-Kontrakt-Abweichung (1 vs. n Orders) | niedrig | hoch | Per Design Single-Order-Return; Caller-Loop fuer Volume-Handling | -| Timestamp-Kollision bei identischem date_created_gmt | niedrig | mittel | Adressiert durch Tupel-Cursor (ts, id) + exclude-Parameter | +| Timestamp-Kollision bei identischem date_created_gmt | niedrig | mittel | Adressiert durch Tupel-Cursor + Bucket-Akkumulation (letzter_import_order_ids sammelt alle IDs des Buckets) | +| Unbounded exclude-Liste bei grossem Bucket | niedrig | niedrig | In der Praxis kleine Listen; URL < 2 KB bei ~100 IDs | ## 9. Definition of Done From 726f2eb2c379da3b01070257f4c777cd47b48b23 Mon Sep 17 00:00:00 2001 From: Avatarsia Date: Tue, 21 Apr 2026 13:55:03 +0200 Subject: [PATCH 18/18] fix(woocommerce): persist fallback cursor when ab_nummer resolution fails If resolveAbNummerToTimestamp() cannot find the referenced legacy order (404, missing date_created_gmt, etc.) the migration previously left the importer on the volatile 30-day fallback, which is recomputed on every run as now()-30d. The cron cycle would then re-scan the same rolling window forever, multiplying API load and caller-dedup activity. On resolution failure we now explicitly persist the current fallback timestamp so the fallback flag flips to false and the lower bound stays stable across runs. Also emits a warning so the operator can spot the stale ab_nummer. Spotted by review of #265. --- www/pages/shopimporter_woocommerce.php | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/www/pages/shopimporter_woocommerce.php b/www/pages/shopimporter_woocommerce.php index 9eccc455a..4f63e047a 100644 --- a/www/pages/shopimporter_woocommerce.php +++ b/www/pages/shopimporter_woocommerce.php @@ -259,7 +259,18 @@ private function migrateAbNummerIfNeeded() $resolved = $this->resolveAbNummerToTimestamp((int) $data['ab_nummer']); if ($resolved !== null) { $this->persistLastImportTimestamp($resolved); + return; } + // Resolution failed (order deleted, 404, missing date_created_gmt). Persist the + // current 30-day fallback so subsequent runs use a stable lower bound + // instead of sliding the window on every cron cycle. + $this->logger->warning( + sprintf( + 'WooCommerce ab_nummer=%d konnte nicht aufgeloest werden; persistiere 30-Tage-Fallback als Cursor', + (int) $data['ab_nummer'] + ) + ); + $this->persistLastImportTimestamp($this->lastImportTimestamp); } // This function searches the wcOrder for the specified WC Meta key