diff --git a/docs-mslearn/toolkit/changelog.md b/docs-mslearn/toolkit/changelog.md index d1f201360..39f22e06c 100644 --- a/docs-mslearn/toolkit/changelog.md +++ b/docs-mslearn/toolkit/changelog.md @@ -3,7 +3,7 @@ title: FinOps toolkit changelog description: Review the latest features and enhancements in the FinOps toolkit, including updates to FinOps hubs, Power BI reports, and more. author: MSBrett ms.author: brettwil -ms.date: 10/28/2026 +ms.date: 10/29/2026 ms.topic: reference ms.service: finops ms.subservice: finops-toolkit @@ -25,6 +25,15 @@ This article summarizes the features and enhancements in each release of the Fin The following section lists features and enhancements that are currently in development. +### [Power BI reports](power-bi/reports.md) + +_**Breaking change**_ + +- **Added** + - Added support for incremental refresh on the **Costs** table in storage reports. Power BI can now detect the **RangeStart**/**RangeEnd** filter on the `x_IncrementalRefreshDate` column. To learn more, see [Configure incremental refresh](power-bi/setup.md#configure-incremental-refresh) ([#2314](https://github.com/microsoft/finops-toolkit/issues/2314)). +- **Changed** + - Changed the `x_IncrementalRefreshDate` column in storage reports from text to Date/Time. Update any custom filters, measures, or visuals that treat it as text or a number ([#2314](https://github.com/microsoft/finops-toolkit/issues/2314)). + -->
diff --git a/docs-mslearn/toolkit/help/data-dictionary.md b/docs-mslearn/toolkit/help/data-dictionary.md index c6c7a50dd..a641d6ee1 100644 --- a/docs-mslearn/toolkit/help/data-dictionary.md +++ b/docs-mslearn/toolkit/help/data-dictionary.md @@ -3,7 +3,7 @@ title: FinOps toolkit data dictionary description: This article describes the column names found in FinOps toolkit solutions, including columns from Cost Management and FOCUS. author: flanakin ms.author: micflan -ms.date: 06/21/2026 +ms.date: 10/29/2026 ms.topic: concept-article ms.service: finops ms.subservice: finops-toolkit @@ -113,7 +113,7 @@ Quick links: | EffectiveCostRunningTotal | Derived. Used in Power BI to calculate the total `EffectiveCost` for the reporting period up to and including the specified day. For instance, the cost for the 15th will include the cost from the 1st through the 15th. | | EffectivePricingCost | `EffectiveCost` in the pricing currency. | | EffectiveUnitPrice | Amortized price per unit after commitment discounts. | -| IncrementalRefreshDate | Derived. Numeric version of the `ChargePeriodStart` column to simplify setup for incremental refresh. | +| IncrementalRefreshDate | Derived. Date/Time version of the charge date (for example, `ChargePeriodStart`) at midnight to simplify setup for incremental refresh. | | InvoiceId | Unique identifier for the invoice the charge is included in. Only available for closed months after the invoice is published. Renamed from **x_InvoiceId** in FOCUS 1.2. | | InvoiceIssuerId | Unique identifier of the organization that generated the invoice. | | InvoiceIssuerName | Name of the organization that generated the invoice. Only supported for CSP accounts. Not supported for EA or MCA accounts that a partner manages due to data not being provided by Cost Management. | diff --git a/docs-mslearn/toolkit/power-bi/setup.md b/docs-mslearn/toolkit/power-bi/setup.md index 5cb617d76..072bd58a6 100644 --- a/docs-mslearn/toolkit/power-bi/setup.md +++ b/docs-mslearn/toolkit/power-bi/setup.md @@ -3,7 +3,7 @@ title: Set up Power BI reports description: Learn how to set up Power BI FinOps reports using the FinOps toolkit, customize visuals, and connect to your cost data for detailed analysis. author: flanakin ms.author: micflan -ms.date: 07/30/2026 +ms.date: 10/29/2026 ms.topic: how-to ms.service: finops ms.subservice: finops-toolkit @@ -105,8 +105,8 @@ The FinOps toolkit Power BI reports include preconfigured visuals, but aren't co 4. Append the container and export path, if applicable. - **Number of Months** – Optional number of closed months you would like to report on if you want to always show a specific number of recent months. If not specified, the report will include all data in storage. - **RangeStart** / **RangeEnd** – Optional date range you would like to limit to. If not specified, the report will include all data in storage. - - We generally recommend leaving these dates empty. They are included to support incremental refresh. - - If you need to configure incremental refresh, consider using [FinOps hubs](../hubs/finops-hubs-overview.md) with Data Explorer instead. + - We generally recommend leaving these dates empty. They are included to support incremental refresh. For steps, see [Configure incremental refresh](#configure-incremental-refresh). + - For larger datasets, consider using [FinOps hubs](../hubs/finops-hubs-overview.md) with Data Explorer instead. - FinOps hubs with Data Explorer offers improved performance and is recommended for anyone monitoring over $100,000 in total spend. - Storage reports only support ~$2 million of data without incremental refresh and ~$2 million per month in raw cost details. To learn more, see [Configure incremental refresh](/power-bi/connect-data/incremental-refresh-configure#define-policy). @@ -121,6 +121,28 @@ If you run into any issues syncing your data, see [Troubleshooting Power BI repo
+## Configure incremental refresh + +Storage reports support incremental refresh on the **Costs** table. Incremental refresh is only needed when you have more cost data than Power BI can load in one refresh. If you're monitoring over $100,000 in total spend, consider [FinOps hubs](../hubs/finops-hubs-overview.md) with Data Explorer instead. + +To configure incremental refresh: + +1. Open the storage report in Power BI Desktop. +2. Select **Transform data** to open Power Query Editor. +3. Set **RangeStart** and **RangeEnd** to a small, recent date range, like the first day of last month and the first day of this month. + - These values only apply in Power BI Desktop. The incremental refresh policy replaces them after you publish. +4. Select **Close & Apply**. +5. In the **Data** pane, right-click the **Costs** table and select **Incremental refresh**. +6. Turn on **Incrementally refresh this table**, set the archive and refresh periods, and then select **Apply**. + - The **Costs** query filters the `x_IncrementalRefreshDate` column by **RangeStart** and **RangeEnd**. +7. Publish the report and refresh the semantic model in the Power BI service. + +Rows without a charge date (`x_IncrementalRefreshDate` is empty) are kept in every partition when a range is set, so they can be counted more than once. These rows are rare, but if you see inflated totals after enabling incremental refresh, check for them. + +If you leave **RangeStart** and **RangeEnd** empty, the **Costs** query doesn't filter by them and the report uses **Number of Months** as before. To learn more, see [Configure incremental refresh](/power-bi/connect-data/incremental-refresh-configure). + +
+ ## Use a SAS token to connect data to a report Shared Access Signature (SAS) tokens allow you to connect to a storage account without end user credentials or setting up a service principal. To connect Power BI reports to your data via SAS tokens: diff --git a/src/power-bi/storage/Shared.Dataset/definition/expressions.tmdl b/src/power-bi/storage/Shared.Dataset/definition/expressions.tmdl index 7febd7971..4b5a66479 100644 --- a/src/power-bi/storage/Shared.Dataset/definition/expressions.tmdl +++ b/src/power-bi/storage/Shared.Dataset/definition/expressions.tmdl @@ -298,7 +298,7 @@ expression ftk_Storage = ``` let segments = if d = null then null else Text.SplitAny(Text.From(d), "/- T"), output = - try Number.From(Date.From( + try DateTime.From(Date.From( if Text.Length(segments{0}) = 4 then segments{0} & "-" & Text.PadStart(segments{1}, 2, "0") & "-" & Text.PadStart(segments{2}, 2, "0") else if Text.Length(segments{2}) = 4 then segments{2} & "-" & Text.PadStart(segments{0}, 2, "0") & "-" & Text.PadStart(segments{1}, 2, "0") else d, @@ -314,11 +314,11 @@ expression ftk_Storage = ``` else if data = "reservationdetails" then [UsageDate] else if data = "reservationtransactions" then [EventDate] else null // Will not support incremental refresh - ) + ), type datetime ), FilterRowsByDate = if (start = null and end = null) then RefreshDate else Table.SelectRows(RefreshDate, each - (start = null or [x_IncrementalRefreshDate] = null or [x_IncrementalRefreshDate] >= Number.From(DateTime.From(start))) - and (end = null or [x_IncrementalRefreshDate] = null or [x_IncrementalRefreshDate] < Number.From(DateTime.From(end))) + (start = null or [x_IncrementalRefreshDate] = null or [x_IncrementalRefreshDate] >= Date.StartOfDay(DateTime.From(start))) + and (end = null or [x_IncrementalRefreshDate] = null or [x_IncrementalRefreshDate] < Date.StartOfDay(DateTime.From(end))) ), // TODO: Try using "_" instead of "FilterRowsByDate" diff --git a/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl b/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl index dd1eccee1..1ae609355 100644 --- a/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl +++ b/src/power-bi/storage/Shared.Dataset/definition/tables/Costs.tmdl @@ -1052,7 +1052,8 @@ table Costs annotation PBI_FormatHint = {"isGeneralNumber":true} column x_IncrementalRefreshDate - dataType: string + dataType: dateTime + formatString: General Date lineageTag: 73487d4f-b0dd-4d41-8d9e-739636dabab8 summarizeBy: none sourceColumn: x_IncrementalRefreshDate @@ -1733,7 +1734,14 @@ table Costs */ // Sort columns alphabetically - Output = Table.ReorderColumns(AHB, List.Sort(Table.ColumnNames(AHB))) + Sorted = Table.ReorderColumns(AHB, List.Sort(Table.ColumnNames(AHB))), + + // Incremental refresh -- Must be the last step so Power BI detects the RangeStart/RangeEnd filter + // Matches the ftk_Storage row filter: whole days, >= start and < end, keeps rows without a date, and keeps all rows when RangeStart/RangeEnd are empty + Output = Table.SelectRows(Sorted, each + (RangeStart = null or [x_IncrementalRefreshDate] = null or [x_IncrementalRefreshDate] >= Date.StartOfDay(DateTime.From(RangeStart))) + and (RangeEnd = null or [x_IncrementalRefreshDate] = null or [x_IncrementalRefreshDate] < Date.StartOfDay(DateTime.From(RangeEnd))) + ) in Output ``` diff --git a/src/power-bi/storage/Shared.Dataset/definition/tables/Prices.tmdl b/src/power-bi/storage/Shared.Dataset/definition/tables/Prices.tmdl index 6afdf1b00..aa780f5c0 100644 --- a/src/power-bi/storage/Shared.Dataset/definition/tables/Prices.tmdl +++ b/src/power-bi/storage/Shared.Dataset/definition/tables/Prices.tmdl @@ -108,7 +108,8 @@ table Prices annotation SummarizationSetBy = Automatic column x_IncrementalRefreshDate - dataType: string + dataType: dateTime + formatString: General Date lineageTag: 55e08448-a8ac-4f16-98f3-915f443d4117 summarizeBy: none sourceColumn: x_IncrementalRefreshDate diff --git a/src/power-bi/storage/Shared.Dataset/definition/tables/ReservationDetails.tmdl b/src/power-bi/storage/Shared.Dataset/definition/tables/ReservationDetails.tmdl index 56741f653..692fe3668 100644 --- a/src/power-bi/storage/Shared.Dataset/definition/tables/ReservationDetails.tmdl +++ b/src/power-bi/storage/Shared.Dataset/definition/tables/ReservationDetails.tmdl @@ -85,7 +85,8 @@ table ReservationDetails annotation PBI_FormatHint = {"isGeneralNumber":true} column x_IncrementalRefreshDate - dataType: string + dataType: dateTime + formatString: General Date lineageTag: 7631c2d6-b330-42de-8588-967752ced645 summarizeBy: none sourceColumn: x_IncrementalRefreshDate diff --git a/src/power-bi/storage/Shared.Dataset/definition/tables/ReservationRecommendations.tmdl b/src/power-bi/storage/Shared.Dataset/definition/tables/ReservationRecommendations.tmdl index 24ece707f..02ac7819d 100644 --- a/src/power-bi/storage/Shared.Dataset/definition/tables/ReservationRecommendations.tmdl +++ b/src/power-bi/storage/Shared.Dataset/definition/tables/ReservationRecommendations.tmdl @@ -3,7 +3,8 @@ table ReservationRecommendations lineageTag: a9b8f96b-84d0-4176-95a1-65eade30a2ef column x_IncrementalRefreshDate - dataType: string + dataType: dateTime + formatString: General Date lineageTag: 0d03437c-a892-4d20-9e24-e1ae25b31e47 summarizeBy: none sourceColumn: x_IncrementalRefreshDate diff --git a/src/power-bi/storage/Shared.Dataset/definition/tables/ReservationTransactions.tmdl b/src/power-bi/storage/Shared.Dataset/definition/tables/ReservationTransactions.tmdl index afcd1ebc1..2b5dabdcf 100644 --- a/src/power-bi/storage/Shared.Dataset/definition/tables/ReservationTransactions.tmdl +++ b/src/power-bi/storage/Shared.Dataset/definition/tables/ReservationTransactions.tmdl @@ -156,7 +156,8 @@ table ReservationTransactions annotation SummarizationSetBy = Automatic column x_IncrementalRefreshDate - dataType: string + dataType: dateTime + formatString: General Date lineageTag: db9b73cb-8a9d-455c-88dd-41a7fd42bb2e summarizeBy: none sourceColumn: x_IncrementalRefreshDate diff --git a/src/powershell/Tests/Lint/PowerBIReports.Tests.ps1 b/src/powershell/Tests/Lint/PowerBIReports.Tests.ps1 index f68677cd4..6039554d2 100644 --- a/src/powershell/Tests/Lint/PowerBIReports.Tests.ps1 +++ b/src/powershell/Tests/Lint/PowerBIReports.Tests.ps1 @@ -97,4 +97,27 @@ Describe 'Demo data source' { $inList = $text -match ('"' + [regex]::Escape($account) + '"') $inList | Should -BeTrue -Because "$account should be in the ftk_DemoFilter storage account list" } +} + +Describe 'Storage incremental refresh' { + BeforeAll { + $tablesDir = Join-Path $PSScriptRoot '../../../power-bi/storage/Shared.Dataset/definition/tables' + } + + It 'Ends the Costs query with a RangeStart/RangeEnd row filter' { + # Power BI only offers incremental refresh when the table's own query filters rows with RangeStart and RangeEnd + $text = Get-Content (Join-Path $tablesDir 'Costs.tmdl') -Raw + $isLastStep = $text -match '(?s)Output\s*=\s*Table\.SelectRows\((?(?:(?!Table\.SelectRows).)*?)\)\s*in\s+Output\s*\r?\n\s*```' + $isLastStep | Should -BeTrue -Because 'the final step of the Costs query must be Table.SelectRows so Power BI detects the filter' + $filter = $Matches['filter'] + $filter | Should -Match '\bRangeStart\b' + $filter | Should -Match '\bRangeEnd\b' + $filter | Should -Match 'x_IncrementalRefreshDate' + } + + It 'Types x_IncrementalRefreshDate as dateTime in <_>' -ForEach @('Costs', 'Prices', 'ReservationDetails', 'ReservationRecommendations', 'ReservationTransactions') { + $text = Get-Content (Join-Path $tablesDir "$_.tmdl") -Raw + $isDateTime = $text -match '(?m)^\tcolumn x_IncrementalRefreshDate\r?\n\t\tdataType: dateTime\b' + $isDateTime | Should -BeTrue -Because 'incremental refresh needs a Date/Time column to compare with RangeStart and RangeEnd' + } } \ No newline at end of file