Support pagination for enumerating deleted items

src/Accounts/Accounts/Accounts.csproj

@@ -19,6 +19,7 @@
   </ItemGroup>
 
   <ItemGroup>
+    <PackageReference Include="Microsoft.Identity.Client" Version="4.70.0" />
     <PackageReference Include="System.Security.Permissions" Version="4.7.0" />
   </ItemGroup>
 

src/DataLakeStore/DataLakeStore.Test/ScenarioTests/AdlsTests.cs

@@ -24,7 +24,8 @@ namespace Microsoft.Azure.Commands.DataLakeStore.Test.ScenarioTests
 
     public class AdlsTests : DataLakeStoreTestRunner
     {
-        private readonly string ResourceGroupLocation = "westus";
+        private readonly string AccountName = "sss-datalakestore-test-c17";
+        private readonly string ResourceGroupLocation = "westus2";
         private readonly string TestFileSystemPermissionResourceGroupLocation = "ukwest";
         private readonly string TestFileSystemResourceGroupLocation = "ukwest";
 
@@ -91,11 +92,18 @@ public void TestNegativeAdlsAccount()
             TestRunner.RunTestScript($"Test-NegativeDataLakeStoreAccount -location '{ResourceGroupLocation}'");
         }
 
+        [Fact]
+        [Trait(Category.AcceptanceType, Category.CheckIn)]
+        public void TestAdlsEnumerateDeletedItem()
+        {
+            TestRunner.RunTestScript($"Test-EnumerateDataLakeStoreDeletedItem -location '{ResourceGroupLocation}' -accountName '{AccountName}'");
+        }
+
         [Fact]
         [Trait(Category.AcceptanceType, Category.CheckIn)]
         public void TestAdlsEnumerateAndRestoreDeletedItem()
         {
-            TestRunner.RunTestScript($"Test-EnumerateAndRestoreDataLakeStoreDeletedItem -location '{ResourceGroupLocation}'");
+            TestRunner.RunTestScript($"Test-EnumerateAndRestoreDataLakeStoreDeletedItem -location '{ResourceGroupLocation} '-accountName '{AccountName}'");
         }
     }
 }

src/DataLakeStore/DataLakeStore.Test/ScenarioTests/AdlsTests.ps1

@@ -1000,101 +1000,168 @@ function CreateAndGetVirtualNetwork ($resourceGroupName, $vnetName, $location =
 
 <#
 .SYNOPSIS
-Tests DataLakeStore deleted items operations (Enumerate, Restore).
+Helper function to create files and folders in DataLakeStore account.
 #>
-function Test-EnumerateAndRestoreDataLakeStoreDeletedItem
+function Create-FilesAndFolders
 {
-	param
+    param
+    (
+        [string]$accountName
+    )
+
+    $folderToCreate1 = "/adlTestDir1"
+    $folderToCreate2 = "/adlTestDir2"
+    $fileToCreate1 = "/adlTestDir1/adlfile1"
+    $fileToCreate2 = "/adlTestDir2/adlfile2"
+
+    $result = New-AzDataLakeStoreItem -Account $accountName -path $folderToCreate1 -Folder
+    Assert-NotNull $result "No value was returned on folder creation"
+
+    $result = New-AzDataLakeStoreItem -Account $accountName -path $folderToCreate2 -Folder
+    Assert-NotNull $result "No value was returned on folder creation"
+
+    $result = New-AzDataLakeStoreItem -Account $accountName -path $fileToCreate1
+    Assert-NotNull $result "No value was returned on empty file creation"
+
+    $result = New-AzDataLakeStoreItem -Account $accountName -path $fileToCreate2
+    Assert-NotNull $result "No value was returned on empty file creation"
+}
+
+<#
+.SYNOPSIS
+Helper function to delete files and folders in DataLakeStore account.
+#>
+function Delete-FilesAndFolders
+{
+    param
+    (
+        [string]$accountName
+    )
+
+	$itemsToDelete = @(
+        "/adlTestDir1/adlfile1",
+        "/adlTestDir2/adlfile2",
+        "/adlTestDir1",
+        "/adlTestDir2"
+    )
+
+    foreach ($item in $itemsToDelete) {
+        if (Test-AzDataLakeStoreItem -Account $accountName -Path $item) {
+            Write-Host "Deleting item: $item"
+            Assert-True { Remove-AzDataLakeStoreItem -Account $accountName -Paths $item -Force -Passthru } "Remove item failed: $item"
+        } else {
+            Write-Host "Item does not exist: $item"
+        }
+    }
+}
+
+<#
+.SYNOPSIS
+Tests DataLakeStore deleted items operations (Enumerate).
+#>
+function Test-EnumerateDataLakeStoreDeletedItem
+{
+    param
     (
         $fileToCopy,
-		$location
+        $location,
+        $accountName
+
     )
 
     if ([string]::IsNullOrEmpty($location))
     {
         $location = Get-Location -providerNamespace "Microsoft.CognitiveServices" -resourceType "accounts" -preferredLocation "West US";
     }
 
-	try
-	{
-		# Creating Account
-		$resourceGroupName = Get-ResourceGroupName
-		$accountName = Get-DataLakeStoreAccountName + "-c12" # testing accountname validation
-		New-AzResourceGroup -Name $resourceGroupName -Location $location
-		$accountCreated = New-AzDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName -Location $location
-
-		Assert-AreEqual $accountName $accountCreated.Name
-		Assert-AreEqual $location $accountCreated.Location
-		Assert-AreEqual "Microsoft.DataLakeStore/accounts" $accountCreated.Type
-		Assert-True {$accountCreated.Id -like "*$resourceGroupName*"}
+    try
+    {
+        Create-FilesAndFolders -accountName $accountName
+        Delete-FilesAndFolders -accountName $accountName
 
-		# In loop to check if account exists
-		for ($i = 0; $i -le 60; $i++)
-		{
-			[array]$accountGet = Get-AzDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName
-			if ($accountGet[0].ProvisioningState -like "Succeeded")
-			{
-				Assert-AreEqual $accountName $accountGet[0].Name
-				Assert-AreEqual $location $accountGet[0].Location
-				Assert-AreEqual "Microsoft.DataLakeStore/accounts" $accountGet[0].Type
-				Assert-True {$accountGet[0].Id -like "*$resourceGroupName*"}
-				break
-			}
+        $out = Get-AzDataLakeStoreDeletedItem -Account $accountName -filter "adlTestDir1" -Count 2
+
+        if ($out.ContinuationToken -ne $null)
+        {
+            $out = Get-AzDataLakeStoreDeletedItem -Account $accountName -filter "adlTestDir1" -Count 2 -ListAfter $out.ContinuationToken
+            Assert-NotNull $out "Failed to get deleted items with continuation token"
+        }
+
+        $out = Get-AzDataLakeStoreDeletedItem -Account $accountName -filter "adlTestDir2" -Count 2
+        Assert-NotNull $out "No deleted items found for adlTestDir2"
+
+        if ($out.ContinuationToken -ne $null)
+        {
+            $out = Get-AzDataLakeStoreDeletedItem -Account $accountName -filter "adlTestDir2" -Count 2 -ListAfter $out.ContinuationToken
+            Assert-NotNull $out "Failed to get deleted items with continuation token"
+        }
+    }
+    finally
+    {
+       # Skip cleanup for existing account and resource group
+		Write-Host "Skipping cleanup for existing account and resource group."	
+    }
+}
 
-			Write-Host "account not yet provisioned. current state: $($accountGet[0].ProvisioningState)"
-			[Microsoft.WindowsAzure.Commands.Utilities.Common.TestMockSupport]::Delay(30000)
-			Assert-False {$i -eq 60} " Data Lake Store account is not in succeeded state even after 30 min."
-		}
+function Restore-DeletedItems {
+    param (
+        [string]$accountName,
+        [array]$items
+    )
 
-		# define all the files and folders
-		$folderToCreate1 = "/adlfolderTest1"
-		$folderToCreate2 = "/adlfolderTest2"
-		$fileToCreate1 = "/adlfolderTest1/adlfile1"
-		$fileToCreate2 = "/adlfolderTest2/adlfile2"
+    foreach ($item in $items) {
+        if ([string]::IsNullOrEmpty($item.TrashDirPath) -or [string]::IsNullOrEmpty($item.OriginalPath)) {
+            continue
+        }
+
+        # Check if the destination already exists
+        $relativeOriginalPath = $item.OriginalPath -replace "^adl://[^/]+", ""
+        if (Test-AzDataLakeStoreItem -Account $accountName -Path $relativeOriginalPath) {
+            continue
+        }
+
+        # Attempt to restore the item
+        Assert-True {
+            Restore-AzDataLakeStoreDeletedItem -Account $accountName -Path $item.TrashDirPath -Destination $item.OriginalPath -Type $item.Type -Force -Passthru
+        }
+    }
+}
 
-		# Create and get Empty folder
-		$result = New-AzDataLakeStoreItem -Account $accountName -path $folderToCreate1 -Folder
-		Assert-NotNull $result "No value was returned on folder creation"
+<#
+.SYNOPSIS
+Tests DataLakeStore deleted items operations (Enumerate, Restore).
+#>
+function Test-EnumerateAndRestoreDataLakeStoreDeletedItem
+{
+	param
+    (
+        $fileToCopy,
+		$location,
+		$accountName
 
-		$result = New-AzDataLakeStoreItem -Account $accountName -path $folderToCreate2 -Folder
-		Assert-NotNull $result "No value was returned on folder creation"
-
-		# Create and get Empty File
-		$result = New-AzDataLakeStoreItem -Account $accountName -path $fileToCreate1
-		Assert-NotNull $result "No value was returned on empty file creation"
-		$result = New-AzDataLakeStoreItem -Account $accountName -path $fileToCreate2
-		Assert-NotNull $result "No value was returned on empty file creation"
-
-	    # delete a file
-		Assert-True {Remove-AzDataLakeStoreItem -Account $accountName -paths $fileToCreate1 -force -passthru } "Remove File Failed"
-		Assert-Throws {Get-AzDataLakeStoreItem -Account $accountName -path $fileToCreate1}
-		Assert-True {Remove-AzDataLakeStoreItem -Account $accountName -paths $fileToCreate2 -force -passthru } "Remove File Failed"
-		Assert-Throws {Get-AzDataLakeStoreItem -Account $accountName -path $fileToCreate2}
-
-		# search delete folder
-		$out = Get-AzDataLakeStoreDeletedItem -Account $accountName -filter "adlfolderTest1" -Count 1000
-		foreach($item in $out)
-		{
-            Restore-AzDataLakeStoreDeletedItem -Account $accountName -Path $item.TrashDirPath -Destination $item.OriginalPath -Type "file" -Force -Passthru
-			Assert-True { Restore-AzDataLakeStoreDeletedItem -Account $accountName -Path $item.TrashDirPath -Destination $item.OriginalPath -Type "file" -Force -Passthru}
-		}
+    )
 
-		$out = Get-AzDataLakeStoreDeletedItem -Account $accountName -filter "adlfolderTest2" -Count 1000
-		foreach($item in $out)
-		{
-			Assert-True { Restore-AzDataLakeStoreDeletedItem -Account $accountName $item -Force -Passthru}
-		}
-
-		# Delete Data Lake account
-		Assert-True {Remove-AzDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName -Force -PassThru} "Remove Account failed."
+    if ([string]::IsNullOrEmpty($location))
+    {
+        $location = Get-Location -providerNamespace "Microsoft.CognitiveServices" -resourceType "accounts" -preferredLocation "West US";
+    }
 
-		# Verify that it is gone by trying to get it again
-		Assert-Throws {Get-AzDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName}
+	try
+	{
+        Create-FilesAndFolders -accountName $accountName
+        Delete-FilesAndFolders -accountName $accountName
+
+		# search deleted folder
+		$out = Get-AzDataLakeStoreDeletedItem -Account $accountName -filter "adlTestDir1" -Count 1000
+		Restore-DeletedItems -accountName $accountName -items $out
+
+		$out = Get-AzDataLakeStoreDeletedItem -Account $accountName -filter "adlTestDir2" -Count 1000
+		 Restore-DeletedItems -accountName $accountName -items $out
 	}
 	finally
 	{
-		# cleanup the resource group that was used in case it still exists. This is a best effort task, we ignore failures here.
-		Invoke-HandledCmdlet -Command {Remove-AzDataLakeStoreAccount -ResourceGroupName $resourceGroupName -Name $accountName -Force -ErrorAction SilentlyContinue} -IgnoreFailures
-		Invoke-HandledCmdlet -Command {Remove-AzResourceGroup -Name $resourceGroupName -Force -ErrorAction SilentlyContinue} -IgnoreFailures
+		Delete-FilesAndFolders -accountName $accountName
+		# Skip cleanup for existing account and resource group
+		Write-Host "Skipping cleanup for existing account and resource group."
 	}
 }

src/DataLakeStore/DataLakeStore.Test/ScenarioTests/DataLakeStoreTestRunner.cs

@@ -17,6 +17,13 @@
 using Microsoft.Azure.Commands.TestFx;
 using System.Collections.Generic;
 using Xunit.Abstractions;
+using Azure.Identity;
+using Azure.Core;
+using System;
+using System.Net.Http.Headers;
+using System.Net.Http;
+using System.Threading.Tasks;
+using System.Threading;
 
 namespace Microsoft.Azure.Commands.DataLake.Test.ScenarioTests
 {
@@ -57,18 +64,43 @@ protected DataLakeStoreTestRunner(ITestOutputHelper output)
                     (ignoreResourcesClient, resourceProviders, userAgentsToIgnore) => new UrlDecodingRecordMatcher(ignoreResourcesClient, resourceProviders, userAgentsToIgnore)
                 )
                 .WithManagementClients(context =>
-                    {
-                        AdlsClientFactory.IsTest = true;
-                        var creds = context.GetClientCredentials(AzureEnvironment.Endpoint.AzureDataLakeStoreFileSystemEndpointSuffix);
-                        AdlsClientFactory.CustomDelegatingHAndler = context.AddHandlers(creds, new AdlMockDelegatingHandler());
-                        AdlsClientFactory.MockCredentials = creds;
-                        return new object();
-                    }
-                )
+                {
+                    AdlsClientFactory.IsTest = true;
+                    var creds = new DefaultAzureCredential(new DefaultAzureCredentialOptions { 
+                        ExcludeSharedTokenCacheCredential = true,
+                        ExcludeVisualStudioCredential = true,
+                        ExcludeVisualStudioCodeCredential = true,
+                        ExcludeAzureCliCredential = true,
+                        ExcludeEnvironmentCredential = true,
+                        ExcludeManagedIdentityCredential = true
+                    });
+
+                    var handlers = context.AddHandlers(new TokenCredentialAdapter(creds), new AdlMockDelegatingHandler());
+                    AdlsClientFactory.CustomDelegatingHAndler = handlers;
+                    AdlsClientFactory.MockCredentials = creds;
+                    return new object();
+                })
                 .WithCleanupAction(
                     () => AdlsClientFactory.IsTest = false
                 )
                 .Build();
         }
     }
-}
+
+    public class TokenCredentialAdapter : Microsoft.Rest.ServiceClientCredentials
+    {
+        private readonly TokenCredential _tokenCredential;
+
+        public TokenCredentialAdapter(TokenCredential tokenCredential)
+        {
+            _tokenCredential = tokenCredential;
+        }
+
+        public override async Task ProcessHttpRequestAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+        {
+            var tokenRequestContext = new TokenRequestContext(new[] { "https://management.azure.com/.default" });
+            var accessToken = await _tokenCredential.GetTokenAsync(tokenRequestContext, cancellationToken).ConfigureAwait(false);
+            request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", accessToken.Token);
+        }
+    }
+}

src/DataLakeStore/DataLakeStore/ChangeLog.md

@@ -18,6 +18,7 @@
         - Additional information about change #1
 -->
 ## Upcoming Release
+* Added support for passing continuation token to enable pagination in GetAzureDataLakeStoreDeletedItem
 
 ## Version 1.4.0
 * Upgraded nuget package to signed package.

src/DataLakeStore/DataLakeStore/DataLakeStore.csproj

@@ -1,23 +1,24 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <PsModuleName>DataLakeStore</PsModuleName>
-    <OmitJsonPackage>true</OmitJsonPackage>
+	<PsModuleName>DataLakeStore</PsModuleName>
+	<OmitJsonPackage>true</OmitJsonPackage>
   </PropertyGroup>
 
   <Import Project="$(MSBuildThisFileDirectory)..\..\Az.props" />
 
   <PropertyGroup>
-    <RootNamespace>$(LegacyAssemblyPrefix)$(PsModuleName)</RootNamespace>
+	<RootNamespace>$(LegacyAssemblyPrefix)$(PsModuleName)</RootNamespace>
   </PropertyGroup>
 
   <ItemGroup>
-    <PackageReference Include="Microsoft.Azure.DataLake.Store" Version="1.2.4-alpha" />
-    <PackageReference Include="Microsoft.Azure.Management.DataLake.Store" Version="2.4.2-preview" />
-    <PackageReference Include="NLog" Version="4.5.0" />
-    <PackageReference Include="System.Net.Requests" Version="4.3.0" />
+	<PackageReference Include="Microsoft.Azure.DataLake.Store" Version="2.0.2" />
+	<PackageReference Include="Microsoft.Azure.Management.DataLake.Store" Version="2.4.2-preview" />
+	<PackageReference Include="NLog" Version="5.4.0" />
+	<PackageReference Include="System.Net.Requests" Version="4.3.0" />
+	<PackageReference Include="Azure.Identity" Version="1.13.2" />
   </ItemGroup>
 
   <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory).., build.proj))\src\Az.Post.props" />
 
-</Project>
+</Project>