From 766902633f8a5e6ed7c8197f398ec50b822f7f7a Mon Sep 17 00:00:00 2001 From: jodobrze Date: Fri, 18 Nov 2022 01:13:01 -0600 Subject: [PATCH 1/4] Updated script to output csv --- SamplesV2/PastRunDetails/PastRunDetails.ps1 | 62 +++++++++------------ 1 file changed, 26 insertions(+), 36 deletions(-) diff --git a/SamplesV2/PastRunDetails/PastRunDetails.ps1 b/SamplesV2/PastRunDetails/PastRunDetails.ps1 index 90b03c80..59d0c049 100644 --- a/SamplesV2/PastRunDetails/PastRunDetails.ps1 +++ b/SamplesV2/PastRunDetails/PastRunDetails.ps1 @@ -1,45 +1,35 @@ - -# This simple script prints pipeline and activity level run details between two dates. -# You can go back 45 days only. You can change output to whatever format (csv) as -# you like with custom printing. This prints same run hour/corehour values as what you see in UX. +$dataFactoryName = "" +$resourceGroupName = "" +$startTime = "11/18/2022 00:00:00" +$endTime = "11/18/2022 12:00:00" +$filePath = "C:\temp\adfouput.csv" -# Here is sample output. Azure IR is used with General Compute meter. It shows corehours and hours spent. -<# ExternalActivity -[ - { - "meterType": "General", - "duration": 0.28871840533333332, - "unit": "coreHour", - "sessionType": "WarmCluster" - } -] -for -executedataflow -[ - { - "meterType": "AzureIR", - "duration": 0.016666666666666666, - "unit": "Hours" - } -] -for -ExternalActivity -#> +$pipelineRuns = Get-AzDataFactoryV2PipelineRun -ResourceGroupName $resourceGroupName -DataFactoryName $dataFactoryName -LastUpdatedAfter $startTime -LastUpdatedBefore $endTime + +$activityDetails = @() -$startTime = "5/5/2021 4:00:00" -$endTime = "5/5/2021 7:00:00" - -$pipelineRuns = Get-AzDataFactoryV2PipelineRun -ResourceGroupName ADF -DataFactoryName adfpupcanary -LastUpdatedAfter $startTime -LastUpdatedBefore $endTime - foreach($pipelineRun in $pipelineRuns) { - $activtiyRuns = Get-AzDataFactoryV2ActivityRun -ResourceGroupName ADF -DataFactoryName adfpupcanary -pipelineRunId $pipelineRun.RunId -RunStartedAfter $startTime -RunStartedBefore $endTime + + $activtiyRuns = Get-AzDataFactoryV2ActivityRun -ResourceGroupName $resourceGroupName -DataFactoryName $dataFactoryName -pipelineRunId $pipelineRun.RunId -RunStartedAfter $startTime -RunStartedBefore $endTime + foreach($activtiyRun in $activtiyRuns) { if ($activtiyRun.Output -ne $null -and $activtiyRun.Output.SelectToken("billingReference.billableDuration") -ne $null) { - Write-Output $activtiyRun.Output.SelectToken("billingReference.billableDuration").ToString() for $activtiyRun.Output.SelectToken("billingReference.activityType").ToString() - } - else { - Write-Output "Not Availble" for $activtiyRun.ActivityType + + $x = @() + $x = $activtiyRun.Output.SelectToken("billingReference.billableDuration").ToString() | ConvertFrom-Json + $x | Add-Member -MemberType NoteProperty -Name "activityBillingType" -Value $activtiyRun.Output.SelectToken("billingReference.activityType").ToString() + $x | Add-Member -MemberType NoteProperty -Name "activityType" -Value $activtiyRun.ActivityType.ToString() + $x | Add-Member -MemberType NoteProperty -Name "activityName" -Value $activtiyRun.ActivityName.ToString() + $x | Add-Member -MemberType NoteProperty -Name "activityRunStart" -Value $activtiyRun.ActivityRunStart.ToString() + $x | Add-Member -MemberType NoteProperty -Name "pipelineRunId" -Value $pipelineRun.RunId + $x | Add-Member -MemberType NoteProperty -Name "pipelineName" -Value $pipelineRun.PipelineName + $x | Add-Member -MemberType NoteProperty -Name "dataFactoryName" -Value $pipelineRun.DataFactoryName + + $activityDetails += $x + } } } + +$activityDetails | Export-Csv -Path $filePath \ No newline at end of file From 6007dc1c846739ba4c9ac5b0ac29a006ce4d4155 Mon Sep 17 00:00:00 2001 From: jodobrze Date: Mon, 6 Feb 2023 13:47:51 -0600 Subject: [PATCH 2/4] Moved new script to separate file. --- SamplesV2/PastRunDetails/PastRunDetails.ps1 | 64 +++++++++++-------- .../PastRunDetails_ActivityLevelCSV.ps1 | 51 +++++++++++++++ SamplesV2/PastRunDetails/README.md | 6 +- 3 files changed, 92 insertions(+), 29 deletions(-) create mode 100644 SamplesV2/PastRunDetails/PastRunDetails_ActivityLevelCSV.ps1 diff --git a/SamplesV2/PastRunDetails/PastRunDetails.ps1 b/SamplesV2/PastRunDetails/PastRunDetails.ps1 index 59d0c049..54aa4bd3 100644 --- a/SamplesV2/PastRunDetails/PastRunDetails.ps1 +++ b/SamplesV2/PastRunDetails/PastRunDetails.ps1 @@ -1,35 +1,45 @@ -$dataFactoryName = "" -$resourceGroupName = "" -$startTime = "11/18/2022 00:00:00" -$endTime = "11/18/2022 12:00:00" -$filePath = "C:\temp\adfouput.csv" + +# This simple script prints pipeline and activity level run details between two dates. +# You can go back 45 days only. You can change output to whatever format (csv) as +# you like with custom printing. This prints same run hour/corehour values as what you see in UX. -$pipelineRuns = Get-AzDataFactoryV2PipelineRun -ResourceGroupName $resourceGroupName -DataFactoryName $dataFactoryName -LastUpdatedAfter $startTime -LastUpdatedBefore $endTime - -$activityDetails = @() +# Here is sample output. Azure IR is used with General Compute meter. It shows corehours and hours spent. +<# ExternalActivity +[ + { + "meterType": "General", + "duration": 0.28871840533333332, + "unit": "coreHour", + "sessionType": "WarmCluster" + } +] +for +executedataflow +[ + { + "meterType": "AzureIR", + "duration": 0.016666666666666666, + "unit": "Hours" + } +] +for +ExternalActivity +#> +$startTime = "5/5/2021 4:00:00" +$endTime = "5/5/2021 7:00:00" + +$pipelineRuns = Get-AzDataFactoryV2PipelineRun -ResourceGroupName ADF -DataFactoryName adfpupcanary -LastUpdatedAfter $startTime -LastUpdatedBefore $endTime + foreach($pipelineRun in $pipelineRuns) { - - $activtiyRuns = Get-AzDataFactoryV2ActivityRun -ResourceGroupName $resourceGroupName -DataFactoryName $dataFactoryName -pipelineRunId $pipelineRun.RunId -RunStartedAfter $startTime -RunStartedBefore $endTime - + $activtiyRuns = Get-AzDataFactoryV2ActivityRun -ResourceGroupName ADF -DataFactoryName adfpupcanary -pipelineRunId $pipelineRun.RunId -RunStartedAfter $startTime -RunStartedBefore $endTime foreach($activtiyRun in $activtiyRuns) { if ($activtiyRun.Output -ne $null -and $activtiyRun.Output.SelectToken("billingReference.billableDuration") -ne $null) { - - $x = @() - $x = $activtiyRun.Output.SelectToken("billingReference.billableDuration").ToString() | ConvertFrom-Json - $x | Add-Member -MemberType NoteProperty -Name "activityBillingType" -Value $activtiyRun.Output.SelectToken("billingReference.activityType").ToString() - $x | Add-Member -MemberType NoteProperty -Name "activityType" -Value $activtiyRun.ActivityType.ToString() - $x | Add-Member -MemberType NoteProperty -Name "activityName" -Value $activtiyRun.ActivityName.ToString() - $x | Add-Member -MemberType NoteProperty -Name "activityRunStart" -Value $activtiyRun.ActivityRunStart.ToString() - $x | Add-Member -MemberType NoteProperty -Name "pipelineRunId" -Value $pipelineRun.RunId - $x | Add-Member -MemberType NoteProperty -Name "pipelineName" -Value $pipelineRun.PipelineName - $x | Add-Member -MemberType NoteProperty -Name "dataFactoryName" -Value $pipelineRun.DataFactoryName - - $activityDetails += $x - + Write-Output $activtiyRun.Output.SelectToken("billingReference.billableDuration").ToString() for $activtiyRun.Output.SelectToken("billingReference.activityType").ToString() + } + else { + Write-Output "Not Availble" for $activtiyRun.ActivityType } } -} - -$activityDetails | Export-Csv -Path $filePath \ No newline at end of file +} \ No newline at end of file diff --git a/SamplesV2/PastRunDetails/PastRunDetails_ActivityLevelCSV.ps1 b/SamplesV2/PastRunDetails/PastRunDetails_ActivityLevelCSV.ps1 new file mode 100644 index 00000000..9978990d --- /dev/null +++ b/SamplesV2/PastRunDetails/PastRunDetails_ActivityLevelCSV.ps1 @@ -0,0 +1,51 @@ +$dataFactoryName = "" +$resourceGroupName = "" +$startTime = "1/12/2023 00:00:00" +$endTime = "1/13/2023 12:00:00" +$filePath = "C:\temp\adfouput.csv" + +$pipelineRuns = Get-AzDataFactoryV2PipelineRun -ResourceGroupName $resourceGroupName -DataFactoryName $dataFactoryName -LastUpdatedAfter $startTime -LastUpdatedBefore $endTime + +$activityDetails = @() + +foreach($pipelineRun in $pipelineRuns) { + + $activtiyRuns = Get-AzDataFactoryV2ActivityRun -ResourceGroupName $resourceGroupName -DataFactoryName $dataFactoryName -pipelineRunId $pipelineRun.RunId -RunStartedAfter $startTime -RunStartedBefore $endTime + + foreach($activtiyRun in $activtiyRuns) { + if ($activtiyRun.Output -ne $null -and + $activtiyRun.Output.SelectToken("billingReference.billableDuration") -ne $null) { + + $billingReference = $activtiyRun.Output.SelectToken("billingReference.billableDuration").ToString() | ConvertFrom-Json + $activityBillingType = $activtiyRun.Output.SelectToken("billingReference.activityType").ToString() + + $activityDetails += @{ + meterType = $billingReference.meterType + duration = $billingReference.duration + unit = $billingReference.unit + activityBillingType = $activityBillingType + activityType = $activtiyRun.ActivityType.ToString() + activityName = $activtiyRun.ActivityName.ToString() + activityRunStart = $activtiyRun.ActivityRunStart.ToString() + pipelineRunId = $pipelineRun.RunId + pipelineName = $pipelineRun.PipelineName + dataFactoryName = $pipelineRun.DataFactoryName + } + + # $x = @() + # $x = $activtiyRun.Output.SelectToken("billingReference.billableDuration").ToString() | ConvertFrom-Json + # $x | Add-Member -MemberType NoteProperty -Name "activityBillingType" -Value $activtiyRun.Output.SelectToken("billingReference.activityType").ToString() + # $x | Add-Member -MemberType NoteProperty -Name "activityType" -Value $activtiyRun.ActivityType.ToString() + # $x | Add-Member -MemberType NoteProperty -Name "activityName" -Value $activtiyRun.ActivityName.ToString() + # $x | Add-Member -MemberType NoteProperty -Name "activityRunStart" -Value $activtiyRun.ActivityRunStart.ToString() + # $x | Add-Member -MemberType NoteProperty -Name "pipelineRunId" -Value $pipelineRun.RunId + # $x | Add-Member -MemberType NoteProperty -Name "pipelineName" -Value $pipelineRun.PipelineName + # $x | Add-Member -MemberType NoteProperty -Name "dataFactoryName" -Value $pipelineRun.DataFactoryName + + # $activityDetails += $x + + } + } +} + +$activityDetails | Export-Csv -Path $filePath \ No newline at end of file diff --git a/SamplesV2/PastRunDetails/README.md b/SamplesV2/PastRunDetails/README.md index dd297d9b..589c42f2 100644 --- a/SamplesV2/PastRunDetails/README.md +++ b/SamplesV2/PastRunDetails/README.md @@ -1,7 +1,9 @@ -## Simple script that prints activity level run details in 45 day range +## PastRunDetails.ps1: Simple script that prints activity level run details in 45 day range -### This is useful for users who want to know which activity ran when for how many hours and corehours. The data can be used: +## PastRunDetails_ActivityLevelCSV.ps1: Script that outputs activity level run details to CSV + +### These are useful for users who want to know which activity ran when for how many hours and corehours. The data can be used: * To understand which activities ran for how long and corehours behind it * To build future consumption models from past runs From 61c5742fb55b82ba8e51a3c8d9ebcf8df901114b Mon Sep 17 00:00:00 2001 From: jodobrze Date: Mon, 6 Feb 2023 13:52:15 -0600 Subject: [PATCH 3/4] Restore original script spacing --- SamplesV2/PastRunDetails/PastRunDetails.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SamplesV2/PastRunDetails/PastRunDetails.ps1 b/SamplesV2/PastRunDetails/PastRunDetails.ps1 index 54aa4bd3..90b03c80 100644 --- a/SamplesV2/PastRunDetails/PastRunDetails.ps1 +++ b/SamplesV2/PastRunDetails/PastRunDetails.ps1 @@ -42,4 +42,4 @@ foreach($pipelineRun in $pipelineRuns) { Write-Output "Not Availble" for $activtiyRun.ActivityType } } -} \ No newline at end of file +} From f89aaf48340d08faeb67839fa08214706337ae41 Mon Sep 17 00:00:00 2001 From: jodobrze Date: Tue, 14 Mar 2023 13:38:40 -0500 Subject: [PATCH 4/4] Removed old commented code --- .../PastRunDetails_ActivityLevelCSV.ps1 | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/SamplesV2/PastRunDetails/PastRunDetails_ActivityLevelCSV.ps1 b/SamplesV2/PastRunDetails/PastRunDetails_ActivityLevelCSV.ps1 index 9978990d..13c37b30 100644 --- a/SamplesV2/PastRunDetails/PastRunDetails_ActivityLevelCSV.ps1 +++ b/SamplesV2/PastRunDetails/PastRunDetails_ActivityLevelCSV.ps1 @@ -1,7 +1,7 @@ $dataFactoryName = "" $resourceGroupName = "" $startTime = "1/12/2023 00:00:00" -$endTime = "1/13/2023 12:00:00" +$endTime = "1/13/2023 00:00:00" $filePath = "C:\temp\adfouput.csv" $pipelineRuns = Get-AzDataFactoryV2PipelineRun -ResourceGroupName $resourceGroupName -DataFactoryName $dataFactoryName -LastUpdatedAfter $startTime -LastUpdatedBefore $endTime @@ -13,8 +13,8 @@ foreach($pipelineRun in $pipelineRuns) { $activtiyRuns = Get-AzDataFactoryV2ActivityRun -ResourceGroupName $resourceGroupName -DataFactoryName $dataFactoryName -pipelineRunId $pipelineRun.RunId -RunStartedAfter $startTime -RunStartedBefore $endTime foreach($activtiyRun in $activtiyRuns) { - if ($activtiyRun.Output -ne $null -and - $activtiyRun.Output.SelectToken("billingReference.billableDuration") -ne $null) { + if ($null -ne $activtiyRun.Output -and + $null -ne $activtiyRun.Output.SelectToken("billingReference.billableDuration")) { $billingReference = $activtiyRun.Output.SelectToken("billingReference.billableDuration").ToString() | ConvertFrom-Json $activityBillingType = $activtiyRun.Output.SelectToken("billingReference.activityType").ToString() @@ -31,19 +31,6 @@ foreach($pipelineRun in $pipelineRuns) { pipelineName = $pipelineRun.PipelineName dataFactoryName = $pipelineRun.DataFactoryName } - - # $x = @() - # $x = $activtiyRun.Output.SelectToken("billingReference.billableDuration").ToString() | ConvertFrom-Json - # $x | Add-Member -MemberType NoteProperty -Name "activityBillingType" -Value $activtiyRun.Output.SelectToken("billingReference.activityType").ToString() - # $x | Add-Member -MemberType NoteProperty -Name "activityType" -Value $activtiyRun.ActivityType.ToString() - # $x | Add-Member -MemberType NoteProperty -Name "activityName" -Value $activtiyRun.ActivityName.ToString() - # $x | Add-Member -MemberType NoteProperty -Name "activityRunStart" -Value $activtiyRun.ActivityRunStart.ToString() - # $x | Add-Member -MemberType NoteProperty -Name "pipelineRunId" -Value $pipelineRun.RunId - # $x | Add-Member -MemberType NoteProperty -Name "pipelineName" -Value $pipelineRun.PipelineName - # $x | Add-Member -MemberType NoteProperty -Name "dataFactoryName" -Value $pipelineRun.DataFactoryName - - # $activityDetails += $x - } } }