From 54f5c65ce116d027c5c278ab2ae2a2ce2021f726 Mon Sep 17 00:00:00 2001 From: Meira Josephy <144697924+mjosephym@users.noreply.github.com> Date: Sun, 22 Dec 2024 18:16:21 +0200 Subject: [PATCH 01/15] toc --- .../kusto/.openpublishing.redirection.json | 5 +++++ data-explorer/kusto/management/toc.yml | 14 ++++++-------- data-explorer/kusto/query/toc.yml | 2 ++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/data-explorer/kusto/.openpublishing.redirection.json b/data-explorer/kusto/.openpublishing.redirection.json index bd63dcaed4..bbb15f0000 100644 --- a/data-explorer/kusto/.openpublishing.redirection.json +++ b/data-explorer/kusto/.openpublishing.redirection.json @@ -65,6 +65,11 @@ "redirect_url": "/kusto/management/show-table-operations-mirroring-exported-artifacts-command", "redirect_document_id": true }, + { + "source_path": "management/database-cursor.md", + "redirect_url": "/kusto/query/database-cursor", + "redirect_document_id": true + }, { "source_path": "management/show-table-mirroring-operations-failures.md", "redirect_url": "/kusto/management/show-table-operations-mirroring-failures-command", diff --git a/data-explorer/kusto/management/toc.yml b/data-explorer/kusto/management/toc.yml index 243f175e76..b3af11c57a 100644 --- a/data-explorer/kusto/management/toc.yml +++ b/data-explorer/kusto/management/toc.yml @@ -816,9 +816,9 @@ items: - name: Ingest from query displayName: .set, .append, .set-or-append, .set-or-replace, add row, append href: data-ingestion/ingest-from-query.md - - name: cancel operation command + - name: .cancel operation command href: cancel-operation-command.md - - name: Inline ingestion + - name: .ingest inline command displayName: .ingest inline href: data-ingestion/ingest-inline.md - name: Streaming ingestion @@ -838,7 +838,7 @@ items: - name: .show streamingingestion failures command displayName: .show streaming ingestion failures href: streaming-ingestion-failures.md - - name: Data operations + - name: .show data operations command displayName: .show data operation command href: show-data-operations.md - name: Data export @@ -868,8 +868,6 @@ items: href: data-export/show-continuous-artifacts.md - name: .show continuous-export failures href: data-export/show-continuous-failures.md - - name: Database cursors - href: database-cursor.md - name: Plugin commands items: - name: .show plugins command @@ -897,7 +895,7 @@ items: items: - name: Estimate table size href: estimate-table-size.md - - name: Journal + - name: .show journal commands href: journal.md - name: Operations items: @@ -926,7 +924,7 @@ items: href: show-table-data-statistics.md - name: Advanced data management items: - - name: Clear cached external artifacts + - name: .clear cluster cache external-artifacts command href: clear-external-artifacts-cache-command.md - name: Delete data href: ../concepts/delete-data.md @@ -936,7 +934,7 @@ items: items: - name: Soft delete overview href: ../concepts/data-soft-delete.md - - name: Soft delete command + - name: .delete table records - soft delete command href: soft-delete-command.md - name: Extents (data shards) items: diff --git a/data-explorer/kusto/query/toc.yml b/data-explorer/kusto/query/toc.yml index 7f763c0884..a5cff9d2a8 100644 --- a/data-explorer/kusto/query/toc.yml +++ b/data-explorer/kusto/query/toc.yml @@ -7,6 +7,8 @@ items: href: t-sql.md - name: Set timeout limits href: ../set-timeout-limits.md + - name: Database cursors + href: database-cursor.md - name: Debug KQL inline Python href: ../debug-inline-python.md - name: Best practices for KQL queries From 4b34298de3410b9319e86b52642e878be1260501 Mon Sep 17 00:00:00 2001 From: Meira Josephy <144697924+mjosephym@users.noreply.github.com> Date: Sun, 22 Dec 2024 19:51:35 +0200 Subject: [PATCH 02/15] edit --- data-explorer/kusto/query/toc.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-explorer/kusto/query/toc.yml b/data-explorer/kusto/query/toc.yml index a5cff9d2a8..4cc3c4b4da 100644 --- a/data-explorer/kusto/query/toc.yml +++ b/data-explorer/kusto/query/toc.yml @@ -8,7 +8,7 @@ items: - name: Set timeout limits href: ../set-timeout-limits.md - name: Database cursors - href: database-cursor.md + href: ../database-cursor.md - name: Debug KQL inline Python href: ../debug-inline-python.md - name: Best practices for KQL queries From 28e488cc1fc7bd94a80cce1b443d7d8882744715 Mon Sep 17 00:00:00 2001 From: Meira Josephy <144697924+mjosephym@users.noreply.github.com> Date: Sun, 22 Dec 2024 19:58:23 +0200 Subject: [PATCH 03/15] edit --- data-explorer/kusto/{management => query}/database-cursor.md | 0 data-explorer/kusto/query/toc.yml | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename data-explorer/kusto/{management => query}/database-cursor.md (100%) diff --git a/data-explorer/kusto/management/database-cursor.md b/data-explorer/kusto/query/database-cursor.md similarity index 100% rename from data-explorer/kusto/management/database-cursor.md rename to data-explorer/kusto/query/database-cursor.md diff --git a/data-explorer/kusto/query/toc.yml b/data-explorer/kusto/query/toc.yml index 4cc3c4b4da..a5cff9d2a8 100644 --- a/data-explorer/kusto/query/toc.yml +++ b/data-explorer/kusto/query/toc.yml @@ -8,7 +8,7 @@ items: - name: Set timeout limits href: ../set-timeout-limits.md - name: Database cursors - href: ../database-cursor.md + href: database-cursor.md - name: Debug KQL inline Python href: ../debug-inline-python.md - name: Best practices for KQL queries From 384a5c108671199c59510560dbe44ac11a52c777 Mon Sep 17 00:00:00 2001 From: Meira Josephy <144697924+mjosephym@users.noreply.github.com> Date: Sun, 22 Dec 2024 22:13:11 +0200 Subject: [PATCH 04/15] edit --- data-explorer/kusto/query/database-cursor.md | 47 +++++++++----------- 1 file changed, 20 insertions(+), 27 deletions(-) diff --git a/data-explorer/kusto/query/database-cursor.md b/data-explorer/kusto/query/database-cursor.md index c195450b4e..eaad2fdb9b 100644 --- a/data-explorer/kusto/query/database-cursor.md +++ b/data-explorer/kusto/query/database-cursor.md @@ -3,7 +3,7 @@ title: Database cursors description: Learn how to use database cursors to query a database multiple times. ms.reviewer: orspodek ms.topic: reference -ms.date: 08/11/2024 +ms.date: 12/22/2024 monikerRange: "azure-data-explorer" --- # Database cursors @@ -15,39 +15,32 @@ A **database cursor** is a database-level object that lets you query a database Database cursors are designed to address two important scenarios: * The ability to repeat the same query multiple times and get the same results, - as long as the query indicates "same data set". + as long as the query indicates "same data set." * The ability to make an "exactly once" query. This query only "sees" the data that a previous query didn't see, because the data wasn't available then. The query lets you iterate, for example, through all the newly arrived data in a table without fear of processing the same record twice or skipping records by mistake. The database cursor is represented in the query language as a scalar value of type -`string`. The actual value should be considered opaque and there's no support -for any operation other than to save its value or use the cursor functions -noted below. +`string`. The actual value should be considered opaque and there's no support for any operation other than to save its value or use the following cursor functions. ## Cursor functions Kusto provides three functions to help implement the two above scenarios: -* [cursor_current()](../query/cursor-current.md): +* [cursor_current()](cursor-current.md): Use this function to retrieve the current value of the database cursor. You can use this value as an argument to the two other functions. -* [cursor_after(rhs:string)](../query/cursor-after-function.md): - This special function can be used on table records that have the - [IngestionTime policy](show-table-ingestion-time-policy-command.md) enabled. It returns +* [cursor_after(rhs:string)](cursor-after-function.md): + This special function can be used on table records that have the [IngestionTime policy](../management/show-table-ingestion-time-policy-command.md) enabled. It returns a scalar value of type `bool` indicating whether the record's `ingestion_time()` database cursor value comes after the `rhs` database cursor value. -* [cursor_before_or_at(rhs:string)](../query/cursor-before-or-at-function.md): - This special function can be used on the table records that have the - [IngestionTime policy](show-table-ingestion-time-policy-command.md) enabled. It returns - a scalar value of type `bool` indicating whether the record's `ingestion_time()` - database cursor value comes before or at the `rhs` database cursor value. +* [cursor_before_or_at(rhs:string)](cursor-before-or-at-function.md): + This special function can be used on the table records that have the [IngestionTime policy](../management/show-table-ingestion-time-policy-command.md) enabled. It returns a scalar value of type `bool` indicating whether the record's `ingestion_time()` database cursor value comes before or at the `rhs` database cursor value. -The two special functions (`cursor_after` and `cursor_before_or_at`) also have -a side-effect: When they're used, Kusto will emit the **current value of the database cursor** to the `@ExtendedProperties` result set of the query. The property name for the cursor is `Cursor`, and its value is a single `string`. +The two special functions (`cursor_after` and `cursor_before_or_at`) also have a side-effect: When they're used, Kusto emits the **current value of the database cursor** to the `@ExtendedProperties` result set of the query. The property name for the cursor is `Cursor`, and its value is a single `string`. For example: @@ -57,20 +50,17 @@ For example: ## Restrictions -Database cursors can only be used with tables for which the -[IngestionTime policy](show-table-ingestion-time-policy-command.md) -has been enabled. Each record in such a table is associated with the -value of the database cursor that was in effect when the record was ingested. -As such, the [ingestion_time()](../query/ingestion-time-function.md) -function can be used. +Database cursors can only be used with tables for which the [IngestionTime policy](../management/show-table-ingestion-time-policy-command.md) is enabled. Each record in such a table is associated with the value of the database cursor that was in effect when the record was ingested. +As such, the [ingestion_time()](../query/ingestion-time-function.md) function can be used. -The database cursor object holds no meaningful value unless the database has at least one table that has an [IngestionTime policy](show-table-ingestion-time-policy-command.md) defined. +The database cursor object holds no meaningful value unless the database has at least one table that has an [IngestionTime policy](../management/show-table-ingestion-time-policy-command.md) defined. This value is guaranteed to update, as-needed by the ingestion history, into such tables and the queries run, that reference such tables. It might, or might not, be updated in other cases. -The ingestion process first commits the data, so that it's available for querying, and only then assigns an actual cursor value to each record. If you attempt to query for data immediately following the ingestion completion using a database cursor, the results might not yet incorporate the last records added, because they haven't yet been assigned the cursor value. Also, retrieving the current database cursor value repeatedly might return the same value, even if ingestion was done in between, because only a cursor commit can update its value. +The ingestion process first commits the data, so that it's available for querying, and only then assigns an actual cursor value to each record. Querying for data immediately after ingestion using a database cursor might not incorporate the last records added because the cursor value wasn't yet assigned. Also, retrieving the current database cursor value repeatedly might return the same value, even if ingestion was done in between, because only a cursor commit can update its value. + +Querying a table based on database cursors is only guaranteed to "work" (providing exactly-once guarantees) if the records are ingested directly into that table. If you use extents commands, such as [.move extents](../management/move-extents.md) or [.replace extents](../management/replace-extents.md) to move data into the table, or if you're using [.rename table](../management/rename-table-command.md), then querying this table using database cursors isn't guaranteed to avoid missing any data. This is because the ingestion time of the records is assigned when initially ingested, and doesn't change during the move extents operation. -Querying a table based on database cursors is only guaranteed to "work" (providing exactly-once guarantees) if the records are ingested directly into that table. If you're using extents commands, such as [move extents](move-extents.md)/[.replace extents](replace-extents.md) to move data into the table, or if you're using [.rename table](rename-table-command.md), then querying this table using database cursors isn't guaranteed to not miss any data. This is because the ingestion time of the records is assigned when initially ingested, and doesn't change during the move extents operation. -Therefore, when the extents are moved into the target table, it's possible that the cursor value assigned to the records in these extents was already processed (and next query by database cursor will miss the new records). +When the extents are moved into the target table, the assigned cursor value might already have been processed, and the next query by the database cursor will miss the new records. ## Example: Processing records exactly once @@ -95,4 +85,7 @@ Employees | where cursor_after('636040929866477946') // -> 636040929866477950 Employees | where cursor_after('636040929866477950') // -> 636040929866479999 Employees | where cursor_after('636040929866479999') // -> 636040939866479000 -``` \ No newline at end of file +``` + +## Related content + From d007034256fbe479d70e55d758044f9f00af3cfd Mon Sep 17 00:00:00 2001 From: Shlomo Sagir <51323195+shsagir@users.noreply.github.com> Date: Mon, 6 Jan 2025 09:44:25 +0200 Subject: [PATCH 05/15] Update data-explorer/kusto/query/database-cursor.md --- data-explorer/kusto/query/database-cursor.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-explorer/kusto/query/database-cursor.md b/data-explorer/kusto/query/database-cursor.md index eaad2fdb9b..4631ca777f 100644 --- a/data-explorer/kusto/query/database-cursor.md +++ b/data-explorer/kusto/query/database-cursor.md @@ -15,7 +15,7 @@ A **database cursor** is a database-level object that lets you query a database Database cursors are designed to address two important scenarios: * The ability to repeat the same query multiple times and get the same results, - as long as the query indicates "same data set." + as long as the query indicates "same data set". * The ability to make an "exactly once" query. This query only "sees" the data that a previous query didn't see, because the data wasn't available then. From 36be115683f6600f72d8efeecf61603f5e6e406b Mon Sep 17 00:00:00 2001 From: Shlomo Sagir <51323195+shsagir@users.noreply.github.com> Date: Mon, 6 Jan 2025 09:52:41 +0200 Subject: [PATCH 06/15] Update data-explorer/kusto/query/database-cursor.md --- data-explorer/kusto/query/database-cursor.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-explorer/kusto/query/database-cursor.md b/data-explorer/kusto/query/database-cursor.md index 4631ca777f..c18290eed5 100644 --- a/data-explorer/kusto/query/database-cursor.md +++ b/data-explorer/kusto/query/database-cursor.md @@ -1,7 +1,7 @@ --- title: Database cursors description: Learn how to use database cursors to query a database multiple times. -ms.reviewer: orspodek +ms.reviewer: alexans ms.topic: reference ms.date: 12/22/2024 monikerRange: "azure-data-explorer" From 6ce33dcd92fa912221fab7dd6b5af1de19d7b8b1 Mon Sep 17 00:00:00 2001 From: Meira Josephy <144697924+mjosephym@users.noreply.github.com> Date: Mon, 6 Jan 2025 22:36:14 +0200 Subject: [PATCH 07/15] commands --- .../kusto/.openpublishing.redirection.json | 6 ++++++ .../kusto/management/capacity-policy.md | 6 +++--- .../data-export/disable-enable-continuous.md | 10 +++++----- .../export-data-to-an-external-table.md | 10 +++++----- .../data-export/export-data-to-sql.md | 12 ++++++------ .../data-export/export-data-to-storage.md | 18 +++++++++--------- .../management/data-ingestion/ingest-inline.md | 7 ++++--- ...-from-storage.md => ingest-into-command.md} | 8 ++++---- data-explorer/kusto/management/toc.yml | 10 +++++----- data-explorer/kusto/query/database-cursor.md | 8 ++++++-- 10 files changed, 53 insertions(+), 42 deletions(-) rename data-explorer/kusto/management/data-ingestion/{ingest-from-storage.md => ingest-into-command.md} (98%) diff --git a/data-explorer/kusto/.openpublishing.redirection.json b/data-explorer/kusto/.openpublishing.redirection.json index bbb15f0000..ac24509ab6 100644 --- a/data-explorer/kusto/.openpublishing.redirection.json +++ b/data-explorer/kusto/.openpublishing.redirection.json @@ -70,6 +70,12 @@ "redirect_url": "/kusto/query/database-cursor", "redirect_document_id": true }, + { + "source_path": "management/data-ingestion/ingest-from-storage.md", + "redirect_url": "/kusto/data-ingestion/ingest-into-command", + "redirect_document_id": true + }, + { "source_path": "management/show-table-mirroring-operations-failures.md", "redirect_url": "/kusto/management/show-table-operations-mirroring-failures-command", diff --git a/data-explorer/kusto/management/capacity-policy.md b/data-explorer/kusto/management/capacity-policy.md index 4164c5e07e..b45d9f3a90 100644 --- a/data-explorer/kusto/management/capacity-policy.md +++ b/data-explorer/kusto/management/capacity-policy.md @@ -1,9 +1,9 @@ --- title: Capacity policy description: Learn how to use the capacity policy to control the compute resources of data management operations on a cluster. -ms.reviewer: orspodek +ms.reviewer: alexans ms.topic: reference -ms.date: 08/11/2024 +ms.date: 01/06/2025 monikerRange: "azure-data-explorer" --- # Capacity policy @@ -236,7 +236,7 @@ The default capacity policy has the following JSON representation: Kusto limits the number of concurrent requests for the following user-initiated commands: * **Ingestions** - * This category includes commands that [ingest from storage](data-ingestion/ingest-from-storage.md), [ingest from a query](data-ingestion/ingest-from-query.md), and [ingest inline](data-ingestion/ingest-inline.md). + * This category includes commands that [ingest from storage](data-ingestion/ingest-into-command.md), [ingest from a query](data-ingestion/ingest-from-query.md), and [ingest inline](data-ingestion/ingest-inline.md). * The limit is as defined by the [ingestion capacity](#ingestion-capacity). * **Purges** * The global limit is currently fixed at one per cluster. diff --git a/data-explorer/kusto/management/data-export/disable-enable-continuous.md b/data-explorer/kusto/management/data-export/disable-enable-continuous.md index 2a8e3d4332..a41f935c50 100644 --- a/data-explorer/kusto/management/data-export/disable-enable-continuous.md +++ b/data-explorer/kusto/management/data-export/disable-enable-continuous.md @@ -3,7 +3,7 @@ title: Enable or disable continuous data export description: This article describes how to disable or enable continuous data export. ms.reviewer: yifats ms.topic: reference -ms.date: 12/01/2024 +ms.date: 01/06/2025 --- # Disable or enable continuous export @@ -39,7 +39,7 @@ The result of the [show continuous export command](show-continuous-export.md) of * [Continuous export overview](continuous-data-export.md) * [External tables](../../query/schema-entities/external-tables.md) -* [Create or alter continuous export](create-alter-continuous.md) -* [Drop continuous export](drop-continuous-export.md) -* [Show continuous export](show-continuous-export.md) -* [Show continuous export failures](show-continuous-failures.md) +* [.create or alter continuous-export](create-alter-continuous.md) +* [.drop continuous-export](drop-continuous-export.md) +* [.show continuous-export](show-continuous-export.md) +* [.show continuous-export failures](show-continuous-failures.md) diff --git a/data-explorer/kusto/management/data-export/export-data-to-an-external-table.md b/data-explorer/kusto/management/data-export/export-data-to-an-external-table.md index 1b143a6480..200bf1a778 100644 --- a/data-explorer/kusto/management/data-export/export-data-to-an-external-table.md +++ b/data-explorer/kusto/management/data-export/export-data-to-an-external-table.md @@ -1,11 +1,11 @@ --- -title: Export data to an external table +title: .export to table description: This article describes Export data to an external table. -ms.reviewer: orspodek +ms.reviewer: alexans ms.topic: reference ms.date: 12/01/2024 --- -# Export data to an external table +# .export to table > [!INCLUDE [applies](../../includes/applies-to-version/applies.md)] [!INCLUDE [fabric](../../includes/applies-to-version/fabric.md)] [!INCLUDE [azure-data-explorer](../../includes/applies-to-version/azure-data-explorer.md)] @@ -149,5 +149,5 @@ If the command is executed asynchronously by using the `async` keyword, the outp * [Continuous data export](continuous-data-export.md) * [Management commands overview](../index.md) * [External tables](../../query/schema-entities/external-tables.md) -* [Export data to SQL](export-data-to-sql.md) -* [Export data to cloud storage](export-data-to-storage.md) +* [.export to SQL](export-data-to-sql.md) +* [.export to storage](export-data-to-storage.md) diff --git a/data-explorer/kusto/management/data-export/export-data-to-sql.md b/data-explorer/kusto/management/data-export/export-data-to-sql.md index 7aac410618..f8a23efff6 100644 --- a/data-explorer/kusto/management/data-export/export-data-to-sql.md +++ b/data-explorer/kusto/management/data-export/export-data-to-sql.md @@ -1,11 +1,11 @@ --- -title: Export data to SQL +title: .export to SQL description: This article describes Export data to SQL. -ms.reviewer: orspodek +ms.reviewer: alexans ms.topic: reference ms.date: 12/01/2024 --- -# Export data to SQL +# .export to SQL > [!INCLUDE [applies](../../includes/applies-to-version/applies.md)] [!INCLUDE [fabric](../../includes/applies-to-version/fabric.md)] [!INCLUDE [azure-data-explorer](../../includes/applies-to-version/azure-data-explorer.md)] @@ -124,6 +124,6 @@ The target table is created if it doesn't exist in the target database. * [Ingest from query](../data-ingestion/ingest-from-query.md) * [Management commands overview](../index.md) -* [Export to an external table](export-data-to-an-external-table.md) -* [Export data to cloud storage](export-data-to-storage.md) -* [Create and alter Azure SQL external tables](../external-sql-tables.md) \ No newline at end of file +* [.export to table](export-data-to-an-external-table.md) +* [.export to storage](export-data-to-storage.md) +* [Create and alter Azure SQL external tables](../external-sql-tables.md) diff --git a/data-explorer/kusto/management/data-export/export-data-to-storage.md b/data-explorer/kusto/management/data-export/export-data-to-storage.md index 0ce7bba2ca..775c0238f9 100644 --- a/data-explorer/kusto/management/data-export/export-data-to-storage.md +++ b/data-explorer/kusto/management/data-export/export-data-to-storage.md @@ -1,11 +1,11 @@ --- -title: Export data to cloud storage +title: .export to storage description: Learn how to export data to cloud storage. -ms.reviewer: orspodek +ms.reviewer: alexans ms.topic: reference -ms.date: 12/01/2024 +ms.date: 01/06/2025 --- -# Export data to cloud storage +# .export to storage > [!INCLUDE [applies](../../includes/applies-to-version/applies.md)] [!INCLUDE [fabric](../../includes/applies-to-version/fabric.md)] [!INCLUDE [azure-data-explorer](../../includes/applies-to-version/azure-data-explorer.md)] @@ -32,7 +32,7 @@ You must have at least [Database Viewer](../../access-control/role-based-access- | *PropertyName*, *PropertyValue* | `string` | | A comma-separated list of key-value property pairs. See [supported properties](#supported-properties).| > [!NOTE] -> We highly recommended exporting data to storage that is co-located in the same region as the database itself. This includes data that is exported so it can be transferred to another cloud service in other regions. Writes should be done locally, while reads can happen remotely. +> We highly recommended exporting data to storage that is colocated in the same region as the database itself. This includes data that is exported so it can be transferred to another cloud service in other regions. Writes should be done locally, while reads can happen remotely. ## Supported properties @@ -187,8 +187,8 @@ On export, Kusto data types are mapped to Parquet data types using the following ## Related content -* [Continuous data export](continuous-data-export.md) -* [Ingest from storage](../data-ingestion/ingest-from-storage.md) * [Management commands overview](../index.md) -* [Export to an external table](export-data-to-an-external-table.md) -* [Export to a SQL table](export-data-to-sql.md) +* [.continuous data export](continuous-data-export.md) +* [.ingest into](../data-ingestion/ingest-into-command.md) +* [.export to table](export-data-to-an-external-table.md) +* [.export to SQL](export-data-to-sql.md) diff --git a/data-explorer/kusto/management/data-ingestion/ingest-inline.md b/data-explorer/kusto/management/data-ingestion/ingest-inline.md index d2cce33475..f9fc83d30f 100644 --- a/data-explorer/kusto/management/data-ingestion/ingest-inline.md +++ b/data-explorer/kusto/management/data-ingestion/ingest-inline.md @@ -1,7 +1,7 @@ --- -title: The ingest inline command (push) +title: .ingest inline command (push) description: This article describes the .ingest inline command (push). -ms.reviewer: orspodek +ms.reviewer: alexans ms.topic: reference ms.date: 08/11/2024 --- @@ -77,5 +77,6 @@ The following command ingests data into a table `Logs` with two columns: `Date` ## Related content -* [Ingest from storage](ingest-from-storage.md) +* [Data formats supported for ingestion](../../ingestion-supported-formats.md) +* [.ingest into (pull data from storage)](ingest-into-command.md) * [Ingest from query](ingest-from-query.md) diff --git a/data-explorer/kusto/management/data-ingestion/ingest-from-storage.md b/data-explorer/kusto/management/data-ingestion/ingest-into-command.md similarity index 98% rename from data-explorer/kusto/management/data-ingestion/ingest-from-storage.md rename to data-explorer/kusto/management/data-ingestion/ingest-into-command.md index b68750e1e0..8790fae6e5 100644 --- a/data-explorer/kusto/management/data-ingestion/ingest-from-storage.md +++ b/data-explorer/kusto/management/data-ingestion/ingest-into-command.md @@ -1,11 +1,11 @@ --- -title: Kusto.ingest into command (pull data from storage) +title: .ingest into command (pull data from storage) description: This article describes The .ingest into command (pull data from storage). -ms.reviewer: orspodek +ms.reviewer: alexans ms.topic: reference ms.date: 11/25/2024 --- -# Ingest from storage +# .ingest into > [!INCLUDE [applies](../../includes/applies-to-version/applies.md)] [!INCLUDE [fabric](../../includes/applies-to-version/fabric.md)] [!INCLUDE [azure-data-explorer](../../includes/applies-to-version/azure-data-explorer.md)] @@ -139,7 +139,7 @@ The following example ingests a single file from Amazon S3 using a [preSigned UR ## Related content * [Data formats supported for ingestion](../../ingestion-supported-formats.md) -* [Inline ingestion](ingest-inline.md) +* [.ingest inline](ingest-inline.md) * [Ingest from query (.set, .append, .set-or-append, .set-or-replace)](ingest-from-query.md) * [.show ingestion failures command](../ingestion-failures.md) * [.show ingestion mapping](../show-ingestion-mapping-command.md) diff --git a/data-explorer/kusto/management/toc.yml b/data-explorer/kusto/management/toc.yml index b3af11c57a..74767c809f 100644 --- a/data-explorer/kusto/management/toc.yml +++ b/data-explorer/kusto/management/toc.yml @@ -808,9 +808,9 @@ items: href: ../ingestion-supported-formats.md - name: Ingestion properties href: ../ingestion-properties.md - - name: Ingest from storage + - name: .ingest-into displayName: .ingest into, pull data, pulling ingestion - href: data-ingestion/ingest-from-storage.md + href: data-ingestion/ingest-into-command.md - name: Ingest from query items: - name: Ingest from query @@ -844,11 +844,11 @@ items: - name: Data export href: data-export/index.md items: - - name: To cloud storage + - name: .export to storage href: data-export/export-data-to-storage.md - - name: To an external table + - name: .export totable href: data-export/export-data-to-an-external-table.md - - name: To an SQL table + - name: .export to SQL href: data-export/export-data-to-sql.md - name: Continuous data export items: diff --git a/data-explorer/kusto/query/database-cursor.md b/data-explorer/kusto/query/database-cursor.md index eaad2fdb9b..188d748de8 100644 --- a/data-explorer/kusto/query/database-cursor.md +++ b/data-explorer/kusto/query/database-cursor.md @@ -1,9 +1,9 @@ --- title: Database cursors description: Learn how to use database cursors to query a database multiple times. -ms.reviewer: orspodek +ms.reviewer: alexans ms.topic: reference -ms.date: 12/22/2024 +ms.date: 01/06/2025 monikerRange: "azure-data-explorer" --- # Database cursors @@ -89,3 +89,7 @@ Employees | where cursor_after('636040929866479999') // -> 636040939866479000 ## Related content +* [cursor_current()](cursor-current.md) +* [cursor_before_or_at()](cursor-before-or-at-function.md) +* [cursor_after()](cursor-after-function.md) +* [IngestionTime policy](../management/show-table-ingestion-time-policy-command.md) From 0941365036eb3166263f56a1d0e9345c181a60c4 Mon Sep 17 00:00:00 2001 From: Meira Josephy <144697924+mjosephym@users.noreply.github.com> Date: Tue, 7 Jan 2025 10:11:08 +0200 Subject: [PATCH 08/15] edits --- data-explorer/kusto/.openpublishing.redirection.json | 2 +- .../kusto/management/data-ingestion/ingest-inline.md | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/data-explorer/kusto/.openpublishing.redirection.json b/data-explorer/kusto/.openpublishing.redirection.json index ac24509ab6..57f6542cd7 100644 --- a/data-explorer/kusto/.openpublishing.redirection.json +++ b/data-explorer/kusto/.openpublishing.redirection.json @@ -72,7 +72,7 @@ }, { "source_path": "management/data-ingestion/ingest-from-storage.md", - "redirect_url": "/kusto/data-ingestion/ingest-into-command", + "redirect_url": "/kusto/management/data-ingestion/ingest-into-command", "redirect_document_id": true }, diff --git a/data-explorer/kusto/management/data-ingestion/ingest-inline.md b/data-explorer/kusto/management/data-ingestion/ingest-inline.md index f9fc83d30f..21dfc6ae40 100644 --- a/data-explorer/kusto/management/data-ingestion/ingest-inline.md +++ b/data-explorer/kusto/management/data-ingestion/ingest-inline.md @@ -1,9 +1,9 @@ --- title: .ingest inline command (push) -description: This article describes the .ingest inline command (push). +description: This article describes the `.ingest` `inline` command (push). ms.reviewer: alexans ms.topic: reference -ms.date: 08/11/2024 +ms.date: 01/07/2025 --- # .ingest inline command (push) @@ -33,7 +33,7 @@ You must have at least [Table Ingestor](../../access-control/role-based-access-c | Name | Type | Required | Description | |--|--|--|--| -| *TableName* | `string` | :heavy_check_mark: | The name of the table into which to ingest data. The table name is always relative to the database in context, and its schema is the schema that will be assumed for the data if no schema mapping object is provided.| +| *TableName* | `string` | :heavy_check_mark: | The name of the table into which to ingest data. The table name is always relative to the database in context. Its schema is the default schema assumed for the data if no schema mapping object is provided.| | *Data* | `string` | :heavy_check_mark: | The data content to ingest. Unless otherwise modified by the ingestion properties, this content is parsed as CSV.| | *IngestionPropertyName*, *IngestionPropertyValue* | `string` | | Any number of [ingestion properties](../../ingestion-properties.md) that affect the ingestion process.| @@ -49,7 +49,7 @@ with an empty (zero-valued) extent ID. |Name |Type |Description | |-----------|----------|--------------------------------------------------------------------------| -|ExtentId |`guid` |The unique identifier for the data shard that was generated by the command| +|ExtentId |`guid` |The unique identifier for the data shard that's generated by the command.| ## Examples From cd56de4bc17aedd00114ccac73f8dd9e8ccbb82a Mon Sep 17 00:00:00 2001 From: Meira Josephy <144697924+mjosephym@users.noreply.github.com> Date: Tue, 7 Jan 2025 12:01:13 +0200 Subject: [PATCH 09/15] datatable syntax, clarification --- data-explorer/kusto/query/datatable-operator.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/data-explorer/kusto/query/datatable-operator.md b/data-explorer/kusto/query/datatable-operator.md index 8c88734843..27b3696c9d 100644 --- a/data-explorer/kusto/query/datatable-operator.md +++ b/data-explorer/kusto/query/datatable-operator.md @@ -3,7 +3,7 @@ title: datatable operator description: Learn how to use the datatable operator to define a table with given schema and data. ms.reviewer: alexans ms.topic: reference -ms.date: 08/11/2024 +ms.date: 01/07/2025 --- # datatable operator @@ -16,7 +16,7 @@ Returns a table whose schema and values are defined in the query itself. ## Syntax -`datatable(` *ColumnName* `:` *ColumnType* [`,` ...]`)` `[` *ScalarValue* [`,` *ScalarValue* ...] `]` +`datatable(` *ColumnName* `:` *ColumnType* [`,` ...]`)` `[` *ScalarValue* [`,` ...] `]` [!INCLUDE [syntax-conventions-note](../includes/syntax-conventions-note.md)] @@ -26,7 +26,7 @@ Returns a table whose schema and values are defined in the query itself. |--|--|--|--| | *ColumnName*| `string` | :heavy_check_mark: | The name for a column. | | *ColumnType* | `string` | :heavy_check_mark: | The type of data in the column.| -| *ScalarValue* | scalar | :heavy_check_mark: | The value to insert into the table. The number of values must be an integer multiple of the columns in the table. The *n*'th value must have a type that corresponds to column *n* % *NumColumns*. | +| *ScalarValue* | scalar | :heavy_check_mark: | The value to insert into the table. The total number of values must be a multiple of the number of columns in the table. Each value is assigned to a column based on its position. Specifically, the *n*'th value is assigned to the column at position *n* % *NumColumns*, where *NumColumns* is the total number of columns. | > [!NOTE] > The column name and column value pairs define the schema for the table. From 30b2ad106691da39c27f0925e407945671f81225 Mon Sep 17 00:00:00 2001 From: Meira Josephy <144697924+mjosephym@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:04:56 +0200 Subject: [PATCH 10/15] simpler example --- .../kusto/query/replace-string-function.md | 29 +++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/data-explorer/kusto/query/replace-string-function.md b/data-explorer/kusto/query/replace-string-function.md index 23023b56b6..7f3dcb25d9 100644 --- a/data-explorer/kusto/query/replace-string-function.md +++ b/data-explorer/kusto/query/replace-string-function.md @@ -3,7 +3,7 @@ title: replace_string() description: Learn how to use the replace_string() function to replace all string matches with another string. ms.reviewer: alexans ms.topic: reference -ms.date: 08/11/2024 +ms.date: 01/07/2025 --- # replace_string() @@ -33,7 +33,32 @@ To replace multiple strings, see [replace_strings()](replace-strings-function.md Returns the *text* after replacing all matches of *lookup* with evaluations of *rewrite*. Matches don't overlap. -## Example +## Examples + +### Replace words in a string + +The following example uses `replace_string()` to replace the word "cat" with the word "hamster" in the `Message` string. + +:::moniker range="azure-data-explorer" +> [!div class="nextstepaction"] +> Run the query +::: moniker-end + +```kusto +print Message="A magic trick can turn a cat into a dog" +| extend Outcome = replace_string( + Message, "cat", "hamster") // Lookup strings +``` + +**Output** + +| Message | Outcome | +|--|--| +| A magic trick can turn a cat into a dog | A magic trick can turn a hamster into a dog | + +### Generate and modify a sequence of numbers + +The following example creates a table with column `x` containing a range of numbers from one to five, incremented by one. It then adds the column `str` using the `strcat()` function to concatenate the string "Number is " with the string representation of the `x` column values. Finally, it adds the `replaced` column where the word "is" is replaced by "was" in the strings from the `str` column. :::moniker range="azure-data-explorer" > [!div class="nextstepaction"] From 177c71d2fa9b34aa9efcb1818dd39b8520939cdc Mon Sep 17 00:00:00 2001 From: Meira Josephy <144697924+mjosephym@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:15:15 +0200 Subject: [PATCH 11/15] edit --- data-explorer/kusto/query/replace-string-function.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-explorer/kusto/query/replace-string-function.md b/data-explorer/kusto/query/replace-string-function.md index 7f3dcb25d9..1e9d2c444d 100644 --- a/data-explorer/kusto/query/replace-string-function.md +++ b/data-explorer/kusto/query/replace-string-function.md @@ -58,7 +58,7 @@ print Message="A magic trick can turn a cat into a dog" ### Generate and modify a sequence of numbers -The following example creates a table with column `x` containing a range of numbers from one to five, incremented by one. It then adds the column `str` using the `strcat()` function to concatenate the string "Number is " with the string representation of the `x` column values. Finally, it adds the `replaced` column where the word "is" is replaced by "was" in the strings from the `str` column. +The following example creates a table with column `x` containing numbers from one to five, incremented by one. It adds the column `str` that concatenates "Number is " with the string representation of the `x` column values using the `strcat()` function. It then adds the `replaced` column where "was" replaces the word "is" in the strings from the `str` column. :::moniker range="azure-data-explorer" > [!div class="nextstepaction"] From ece383c0e47d23d43740046bc874685ac57e9ed4 Mon Sep 17 00:00:00 2001 From: Meira Josephy <144697924+mjosephym@users.noreply.github.com> Date: Tue, 7 Jan 2025 15:43:56 +0200 Subject: [PATCH 12/15] stops --- data-explorer/kusto/query/range-operator.md | 50 +++++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/data-explorer/kusto/query/range-operator.md b/data-explorer/kusto/query/range-operator.md index 90725fd23d..b58169d1e7 100644 --- a/data-explorer/kusto/query/range-operator.md +++ b/data-explorer/kusto/query/range-operator.md @@ -3,7 +3,7 @@ title: range operator description: Learn how to use the range operator to generate a single-column table of values. ms.reviewer: alexans ms.topic: reference -ms.date: 08/11/2024 +ms.date: 01/07/2025 --- # range operator @@ -39,6 +39,8 @@ whose values are *start*, *start* `+` *step*, ... up to and until *stop*. ## Examples +### Range over the past seven days + The following example creates a table with entries for the current time stamp extended over the past seven days, once a day. :::moniker range="azure-data-explorer" @@ -59,6 +61,37 @@ range LastWeek from ago(7d) to now() step 1d |...| |2015-12-12 09:10:04.627| +### Combine different stop times + +The following example shows how to extend ranges to use multiple stop times by using the `union` operator. + +:::moniker range="azure-data-explorer" +> [!div class="nextstepaction"] +> Run the query +::: moniker-end + +```kusto +let Range1 = range Time from datetime(2024-01-01) to datetime(2024-01-05) step 1d; +let Range2 = range Time from datetime(2024-01-06) to datetime(2024-01-10) step 1d; +union Range1, Range2 +| order by Time asc + +``` + +**Output** + +| Time | +|--| +| 2024-01-04 00:00:00.0000000 | +| 2024-01-05 00:00:00.0000000 | +| 2024-01-06 00:00:00.0000000 | +| 2024-01-07 00:00:00.0000000 | +| 2024-01-08 00:00:00.0000000 | +| 2024-01-09 00:00:00.0000000 | +| 2024-01-10 00:00:00.0000000 | + +### Range using parameters + The following example shows how to use the `range` operator with parameters, which are then extended and consumed as a table. :::moniker range="azure-data-explorer" @@ -89,8 +122,10 @@ let MyTimeline = range MyMonthHour from MyMonthStart to now() step StepBy | 2023-02-01 | 13:36:07.2000000 | 1675258567.2 | 2023-02-01 00:00:00.0000000 | | ... | ... | ... | ... | +### Incremented steps + The following example creates a table with a single column called `Steps` -whose type is `long` and whose values are `1`, `4`, and `7`. +whose type is `long` and results in values from one to eight incremented by three. :::moniker range="azure-data-explorer" > [!div class="nextstepaction"] @@ -100,8 +135,17 @@ whose type is `long` and whose values are `1`, `4`, and `7`. ```kusto range Steps from 1 to 8 step 3 ``` +**Output** + +| Steps | +|-------| +| 1 | +| 4 | +| 7 | + +### Traces over a time range -The following example shows how the `range` operator can be used to create a small, ad-hoc, dimension table that is then used to introduce zeros where the source data has no values. +The following example shows how the `range` operator can be used to create a small, ad-hoc, dimension table that is then used to introduce zeros where the source data has no values. It takes timestamps from the last 4 hours and counts traces for each 1-minute interval. When no traces exist for a specific interval, the count is 0. ```kusto range TIMESTAMP from ago(4h) to now() step 1m From 3a5c21e3048441c423242b0cec994d5c111cffcd Mon Sep 17 00:00:00 2001 From: Meira Josephy <144697924+mjosephym@users.noreply.github.com> Date: Tue, 7 Jan 2025 16:01:46 +0200 Subject: [PATCH 13/15] edits --- data-explorer/kusto/query/range-operator.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data-explorer/kusto/query/range-operator.md b/data-explorer/kusto/query/range-operator.md index b58169d1e7..ac7487ad25 100644 --- a/data-explorer/kusto/query/range-operator.md +++ b/data-explorer/kusto/query/range-operator.md @@ -145,7 +145,7 @@ range Steps from 1 to 8 step 3 ### Traces over a time range -The following example shows how the `range` operator can be used to create a small, ad-hoc, dimension table that is then used to introduce zeros where the source data has no values. It takes timestamps from the last 4 hours and counts traces for each 1-minute interval. When no traces exist for a specific interval, the count is 0. +The following example shows how the `range` operator can be used to create a dimension table that is used to introduce zeros where the source data has no values. It takes timestamps from the last four hours and counts traces for each one minute interval. When there are no traces for a specific interval, the count is zero. ```kusto range TIMESTAMP from ago(4h) to now() step 1m From 286814016e5c3c18a66115b81180e9016103f681 Mon Sep 17 00:00:00 2001 From: Shlomo Sagir <51323195+shsagir@users.noreply.github.com> Date: Wed, 8 Jan 2025 09:18:09 +0200 Subject: [PATCH 14/15] Update data-explorer/kusto/query/range-operator.md --- data-explorer/kusto/query/range-operator.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/data-explorer/kusto/query/range-operator.md b/data-explorer/kusto/query/range-operator.md index ac7487ad25..0f5418dcf4 100644 --- a/data-explorer/kusto/query/range-operator.md +++ b/data-explorer/kusto/query/range-operator.md @@ -134,8 +134,6 @@ whose type is `long` and results in values from one to eight incremented by thre ```kusto range Steps from 1 to 8 step 3 -``` -**Output** | Steps | |-------| From 0662f87c37dd19e654b7c4f90503dd563ef29622 Mon Sep 17 00:00:00 2001 From: Shlomo Sagir <51323195+shsagir@users.noreply.github.com> Date: Wed, 8 Jan 2025 11:42:15 +0200 Subject: [PATCH 15/15] Update manage-cluster-choose-sku.md --- data-explorer/manage-cluster-choose-sku.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data-explorer/manage-cluster-choose-sku.md b/data-explorer/manage-cluster-choose-sku.md index a3afc15af2..93677ece8f 100644 --- a/data-explorer/manage-cluster-choose-sku.md +++ b/data-explorer/manage-cluster-choose-sku.md @@ -95,7 +95,7 @@ The following SKU series are available for Azure Data Explorer cluster VMs. The |--|--|--|--| | [Lasv3](/azure/virtual-machines/lasv3-series) | 8, 16 , 32| AMD | No | | [Lsv3](/azure/virtual-machines/lsv3-series) | 8, 16 , 32| Intel | No | -| [Easv4](/azure/virtual-machines/eav4-easv4-series), [Easv5](/azure/virtual-machines/sizes/memory-optimized/eadsv5-series), ECasv5| 8, 16 | AMD | Yes | +| [Easv4](/azure/virtual-machines/eav4-easv4-series), [Easv5](/azure/virtual-machines/sizes/memory-optimized/eadsv5-series), [ECadsv5](/azure/virtual-machines/ecasv5-ecadsv5-series#ecasv5-series)| 8, 16 | AMD | Yes | | [Esv4](/azure/virtual-machines/ev4-esv4-series), [Esv5](/azure/virtual-machines/ev5-esv5-series) | 8, 16 | Intel | Yes | | [DSv2](/azure/virtual-machines/dv2-dsv2-series) | 8, 16 | Intel | Yes | @@ -103,7 +103,7 @@ The following SKU series are available for Azure Data Explorer cluster VMs. The | SKU Series | Available vCPU config | SKU type | Supports premium storage | |--|--|--|--| -| [Eadsv5](/azure/virtual-machines/sizes/memory-optimized/eadsv5-series), [ECadsv5](/azure/virtual-machines/ecasv5-ecadsv5-series)| 2, 4, 8, 16 | AMD | No | +| [Eadsv5](/azure/virtual-machines/sizes/memory-optimized/eadsv5-series), [ECadsv5](/azure/virtual-machines/ecasv5-ecadsv5-series#ecasv5-series)| 2, 4, 8, 16 | AMD | No | | [Edv4](/azure/virtual-machines/edv4-edsv4-series), [Edv5](/azure/virtual-machines/edv5-edsv5-series) | 2, 4, 8, 16 | Intel | No | | [Eav4](/azure/virtual-machines/eav4-easv4-series) | 2, 4, 8, 16 | AMD | No | | [Dv2](/azure/virtual-machines/dv2-dsv2-series) | 2, 4, 8, 16 | Intel | No |