Skip to content

Commit bb49ad1

Browse files
weshaggardweshaggard
authored
Optimize full package discovery (Azure#49787)
1 parent c0aea70 commit bb49ad1

File tree

4 files changed

+86
-93
lines changed

4 files changed

+86
-93
lines changed

‎eng/Directory.Build.Common.targets

Lines changed: 21 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,24 +18,27 @@
1818
Text="Api Compat cannot be disabled."/>
1919
</Target>
2020

21+
<PropertyGroup>
22+
<PackageRootDirectory>$([MSBuild]::NormalizeDirectory($(MSBuildProjectDirectory)/../).TrimEnd("/").TrimEnd("\\"))</PackageRootDirectory>
23+
</PropertyGroup>
24+
2125
<!-- Set PackageProjectUrl and PackageReleaseNotes to the package README.md and CHANGELOG.md respectively for DataPlane Libraries -->
2226
<Target Name="SetPackageProjectUrlandReleaseNotes" BeforeTargets="GenerateNuspec" DependsOnTargets="InitializeSourceControlInformationFromSourceControlManager" Condition="'$(IsShippingLibrary)' == 'true' and '$(SourceRevisionId)' != ''">
2327
<Error Condition="'$(IsClientLibrary)' == 'true' and '$(PackageReleaseNotes)' != ''" Text="Do NOT set PackageReleaseNotes property in the project. Release notes are added automatically from package changelog" />
2428
<PropertyGroup>
25-
<PackageRootPath>$([MSBuild]::NormalizeDirectory($(MSBuildProjectDirectory)/../))</PackageRootPath>
26-
<DirectoryPartofPath>$(PackageRootPath.Replace($(RepoRoot), ''))</DirectoryPartofPath>
27-
<PackageProjectUrl Condition="Exists('$(PackageRootPath)/README.md') and '$(SkipDevBuildNumber)' != 'true'">$([System.UriBuilder]::new($(RepositoryUrl)/blob/$(SourceRevisionId)/$(DirectoryPartofPath)README.md).Uri)</PackageProjectUrl>
28-
<PackageProjectUrl Condition="Exists('$(PackageRootPath)/README.md') and '$(SkipDevBuildNumber)' == 'true'">$([System.UriBuilder]::new($(RepositoryUrl)/blob/$(PackageId)_$(Version)/$(DirectoryPartofPath)README.md).Uri)</PackageProjectUrl>
29-
<PackageReleaseNotes Condition="Exists('$(PackageRootPath)/CHANGELOG.md') and '$(SkipDevBuildNumber)' != 'true'">$([System.UriBuilder]::new($(RepositoryUrl)/blob/$(SourceRevisionId)/$(DirectoryPartofPath)CHANGELOG.md).Uri)</PackageReleaseNotes>
30-
<PackageReleaseNotes Condition="Exists('$(PackageRootPath)/CHANGELOG.md') and '$(SkipDevBuildNumber)' == 'true'">$([System.UriBuilder]::new($(RepositoryUrl)/blob/$(PackageId)_$(Version)/$(DirectoryPartofPath)CHANGELOG.md).Uri)</PackageReleaseNotes>
29+
<DirectoryPartofPath>$(PackageRootDirectory.Replace($(RepoRoot), ''))</DirectoryPartofPath>
30+
<PackageProjectUrl Condition="Exists('$(PackageRootDirectory)/README.md') and '$(SkipDevBuildNumber)' != 'true'">$([System.UriBuilder]::new($(RepositoryUrl)/blob/$(SourceRevisionId)/$(DirectoryPartofPath)README.md).Uri)</PackageProjectUrl>
31+
<PackageProjectUrl Condition="Exists('$(PackageRootDirectory)/README.md') and '$(SkipDevBuildNumber)' == 'true'">$([System.UriBuilder]::new($(RepositoryUrl)/blob/$(PackageId)_$(Version)/$(DirectoryPartofPath)README.md).Uri)</PackageProjectUrl>
32+
<PackageReleaseNotes Condition="Exists('$(PackageRootDirectory)/CHANGELOG.md') and '$(SkipDevBuildNumber)' != 'true'">$([System.UriBuilder]::new($(RepositoryUrl)/blob/$(SourceRevisionId)/$(DirectoryPartofPath)CHANGELOG.md).Uri)</PackageReleaseNotes>
33+
<PackageReleaseNotes Condition="Exists('$(PackageRootDirectory)/CHANGELOG.md') and '$(SkipDevBuildNumber)' == 'true'">$([System.UriBuilder]::new($(RepositoryUrl)/blob/$(PackageId)_$(Version)/$(DirectoryPartofPath)CHANGELOG.md).Uri)</PackageReleaseNotes>
3134
</PropertyGroup>
3235
</Target>
3336

3437
<!-- Validates that there is a release note entry for the current package version -->
3538
<Target Name="ValidateReleaseNotes" BeforeTargets="SetPackageProjectUrlandReleaseNotes" Condition="'$(IsShippingLibrary)' == 'true'" >
3639
<PropertyGroup>
3740
<ValidateReleaseNotesScriptPath Condition=" '$(ValidateReleaseNotesScriptPath)'=='' ">$(MSBuildThisFileDirectory)common/scripts/Verify-ChangeLog.ps1</ValidateReleaseNotesScriptPath>
38-
<ChangeLogPath>$([MSBuild]::NormalizeDirectory($(MSBuildProjectDirectory)/../))CHANGELOG.md</ChangeLogPath>
41+
<ChangeLogPath>$(PackageRootDirectory)/CHANGELOG.md</ChangeLogPath>
3942
</PropertyGroup>
4043
<Exec ContinueOnError="true" ConsoleToMSBuild="true" StandardOutputImportance="High" Command="$(PowerShellExe) -NoProfile -NonInteractive -executionpolicy Unrestricted -File $(ValidateReleaseNotesScriptPath) -ChangeLogLocation $(ChangeLogPath) -VersionString $(_VersionInProject)">
4144
<Output TaskParameter="ExitCode" PropertyName="SetReleaseNotesErrorCode" />
@@ -52,7 +55,7 @@
5255
</Target>
5356

5457
<PropertyGroup>
55-
<OriginalReadmeMdPath>$([MSBuild]::NormalizeDirectory($(MSBuildProjectDirectory)/../))README.md</OriginalReadmeMdPath>
58+
<OriginalReadmeMdPath>$(PackageRootDirectory)/README.md</OriginalReadmeMdPath>
5659
<ProcessedReadmeMdPath>$(IntermediateOutputPath)README.md</ProcessedReadmeMdPath>
5760
<EnableNuGetReadmeMd Condition="'$(DesignTimeBuild)' != 'true' and Exists('$(OriginalReadmeMdPath)')">true</EnableNuGetReadmeMd>
5861
<PackageReadmeFile Condition="'$(EnableNuGetReadmeMd)' == 'true'">README.md</PackageReadmeFile>
@@ -113,7 +116,7 @@
113116
<!-- Add Package Icon to DataPlane Packages -->
114117
<ItemGroup Condition="'$(IsTestProject)' != 'true'">
115118
<None Include="$(PackageIconPath)" Pack="true" PackagePath=""/>
116-
<None Condition="Exists('$(MSBuildProjectDirectory)/../CHANGELOG.md')" Include="$(MSBuildProjectDirectory)/../CHANGELOG.md" Pack="true" PackagePath=""/>
119+
<None Condition="Exists('$(PackageRootDirectory)/CHANGELOG.md')" Include="$(PackageRootDirectory)/CHANGELOG.md" Pack="true" PackagePath=""/>
117120
</ItemGroup>
118121

119122
<!-- Add App.config to enable server GC in net462 perf and stress projects -->
@@ -382,8 +385,6 @@
382385
<PackageSdkType Condition="'$(IsFunctionsLibrary)' == 'true'">functions</PackageSdkType>
383386
<PackageSdkType Condition="'$(IsMgmtLibrary)' == 'true'">mgmt</PackageSdkType>
384387

385-
<PackageRootDirectory>$([MSBuild]::NormalizeDirectory($(MSBuildProjectDirectory)/../).TrimEnd("/").TrimEnd("\\"))</PackageRootDirectory>
386-
387388
<!-- If the AddDevVersion property is true this will output the generated dev version (e.g. 1.0.0-alpha.20210921.1) -->
388389
<VersionForProperties>$(_VersionInProject)</VersionForProperties>
389390
<VersionForProperties Condition="'$(AddDevVersion)' == 'true'">$(Version)</VersionForProperties>
@@ -420,12 +421,18 @@
420421

421422
<Target Name="ProjectDependsOnInner" DependsOnTargets="ResolveReferences" Returns="@(ProjectsToInclude)">
422423
<ItemGroup>
424+
<_TestDependsOnDendencies Include="$(TestDependsOnDependency.Split(' '))" />
423425
<_LibrariesReferenced Include="@(ReferencePath -> '%(Filename)')"/>
424-
<_DependsOnGiven Include="@(_LibrariesReferenced)" Condition="'%(Identity)' == '$(TestDependsOnDependency)'" />
426+
<_DependsOnGiven Include="@(_TestDependsOnDendencies)" Exclude="@(_LibrariesReferenced)'" />
425427
</ItemGroup>
426428

427-
<ItemGroup Condition="'@(_DependsOnGiven)' != '' and '$(IsClientLibrary)' == 'true'" >
428-
<ProjectsToInclude Include="$(MSBuildProjectFullPath)" />
429+
<PropertyGroup>
430+
<_ProjectName>%24%28RepoRoot%29$(MSBuildProjectFullPath.Replace($(RepoRoot), ''))</_ProjectName>
431+
<_ProjectName Condition="'$(TestDependsIncludePackageRootDirectoryOnly)' == 'true'">$(PackageRootDirectory)</_ProjectName>
432+
</PropertyGroup>
433+
434+
<ItemGroup Condition="'@(_DependsOnGiven)' != '@(_TestDependsOnDendencies)' and '$(IsClientLibrary)' == 'true'" >
435+
<ProjectsToInclude Include="$(_ProjectName)" />
429436
</ItemGroup>
430437
</Target>
431438

‎eng/pipelines/templates/steps/pr-matrix-presteps.yml

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,10 +11,6 @@ parameters:
1111
default: []
1212

1313
steps:
14-
- script: |
15-
echo "##vso[task.setvariable variable=DISCOVER_DEPENDENT_PACKAGES]1"
16-
displayName: Enable AdditionalDependency Calculation
17-
1814
- template: /eng/pipelines/templates/steps/install-dotnet.yml
1915

2016
- template: /eng/common/pipelines/templates/steps/save-package-properties.yml

‎eng/scripts/Language-Settings.ps1

Lines changed: 62 additions & 72 deletions
Original file line numberDiff line numberDiff line change
@@ -9,32 +9,6 @@ $PackageRepositoryUri = "https://www.nuget.org/packages"
99

1010
. "$PSScriptRoot/docs/Docs-ToC.ps1"
1111

12-
$DependencyCalculationPackages = @(
13-
"Azure.Core",
14-
"Azure.ResourceManager",
15-
"System.ClientModel"
16-
)
17-
18-
function processTestProject($projPath) {
19-
$cleanPath = $projPath -replace "^\$\(RepoRoot\)", ""
20-
21-
# Split the path into segments
22-
$pathSegments = $cleanPath -split "[\\/]"
23-
24-
# Find the index of the 'tests' directory
25-
$testsIndex = $pathSegments.IndexOf("tests")
26-
27-
if ($testsIndex -gt 0) {
28-
# Reconstruct the path up to the directory just above 'tests'
29-
$parentDirectory = ($pathSegments[0..($testsIndex - 1)] -join [System.IO.Path]::DirectorySeparatorChar)
30-
return $parentDirectory
31-
} else {
32-
Write-Error "Unable to retrieve a package directory for this this test project: $projPath"
33-
# If 'tests' is not found, return the original path (optional behavior)
34-
$_
35-
}
36-
}
37-
3812
function Get-AllPackageInfoFromRepo($serviceDirectory)
3913
{
4014
$allPackageProps = @()
@@ -55,7 +29,7 @@ function Get-AllPackageInfoFromRepo($serviceDirectory)
5529
foreach ($projectOutput in $msbuildOutput)
5630
{
5731
if (!$projectOutput) {
58-
Write-Host "Get-AllPackageInfoFromRepo::projectOutput was null or empty, skipping"
32+
Write-Verbose "Get-AllPackageInfoFromRepo::projectOutput was null or empty, skipping"
5933
continue
6034
}
6135

@@ -72,51 +46,6 @@ function Get-AllPackageInfoFromRepo($serviceDirectory)
7246
$pkgProp.IncludedForValidation = $false
7347
$pkgProp.DirectoryPath = ($pkgProp.DirectoryPath)
7448

75-
if ($pkgProp.Name -in $DependencyCalculationPackages -and $env:DISCOVER_DEPENDENT_PACKAGES) {
76-
Write-Host "Calculating dependencies for $($pkgProp.Name)"
77-
$outputFilePath = Join-Path $RepoRoot "$($pkgProp.Name)_dependencylist.txt"
78-
$buildOutputPath = Join-Path $RepoRoot "$($pkgProp.Name)_dependencylistoutput.txt"
79-
80-
if (!(Test-Path $outputFilePath)) {
81-
try {
82-
$command = "dotnet build /t:ProjectDependsOn ./eng/service.proj /p:TestDependsOnDependency=`"$($pkgProp.Name)`" /p:IncludeSrc=false " +
83-
"/p:IncludeStress=false /p:IncludeSamples=false /p:IncludePerf=false /p:RunApiCompat=false /p:InheritDocEnabled=false /p:BuildProjectReferences=false" +
84-
" /p:OutputProjectFilePath=`"$outputFilePath`" > $buildOutputPath 2>&1"
85-
86-
Invoke-LoggedCommand $command | Out-Null
87-
}
88-
catch {
89-
Write-Host "Failed calculating dependencies for $($pkgProp.Name). Exit code $LASTEXITCODE."
90-
Write-Host "Dumping erroring build output."
91-
Write-Host (Get-Content -Raw $buildOutputPath)
92-
continue
93-
}
94-
}
95-
96-
$pkgRelPath = $pkgProp.DirectoryPath.Replace($RepoRoot, "").TrimStart("\/")
97-
98-
if (Test-Path $outputFilePath) {
99-
$dependentProjects = Get-Content $outputFilePath
100-
$testPackages = @{}
101-
102-
foreach ($projectLine in $dependentProjects) {
103-
$testPackage = processTestProject($projectLine)
104-
if ($testPackage -and $testPackage -ne $pkgRelPath) {
105-
if ($testPackages[$testPackage]) {
106-
$testPackages[$testPackage] += 1
107-
}
108-
else {
109-
$testPackages[$testPackage] = 1
110-
}
111-
}
112-
}
113-
114-
$pkgProp.AdditionalValidationPackages = $testPackages.Keys
115-
116-
Write-Host "Cleaning up $outputFilePath."
117-
}
118-
}
119-
12049
$ciProps = $pkgProp.GetCIYmlForArtifact()
12150

12251
if ($ciProps) {
@@ -166,6 +95,67 @@ function Get-AllPackageInfoFromRepo($serviceDirectory)
16695
return $allPackageProps
16796
}
16897

98+
function Get-dotnet-AdditionalValidationPackagesFromPackageSet($LocatedPackages, $diffObj, $AllPkgProps)
99+
{
100+
$additionalValidationPackages = @()
101+
102+
$DependencyCalculationPackages = @(
103+
"Azure.Core",
104+
"Azure.ResourceManager",
105+
"System.ClientModel"
106+
)
107+
108+
$TestDependsOnDependencySet = $LocatedPackages | Where-Object { $_.Name -in $DependencyCalculationPackages }
109+
$TestDependsOnDependency = $TestDependsOnDependencySet.Name -join " "
110+
111+
if (!$TestDependsOnDependency) {
112+
return $additionalValidationPackages
113+
}
114+
115+
Write-Host "Calculating dependencies for $($pkgProp.Name)"
116+
117+
$outputFilePath = Join-Path $RepoRoot "_dependencylist.txt"
118+
$buildOutputPath = Join-Path $RepoRoot "_dependencylistoutput.txt"
119+
120+
try {
121+
$command = "dotnet build /t:ProjectDependsOn ./eng/service.proj /p:TestDependsOnDependency=`"$TestDependsOnDependency`" /p:TestDependsIncludePackageRootDirectoryOnly=true /p:IncludeSrc=false " +
122+
"/p:IncludeStress=false /p:IncludeSamples=false /p:IncludePerf=false /p:RunApiCompat=false /p:InheritDocEnabled=false /p:BuildProjectReferences=false" +
123+
" /p:OutputProjectFilePath=`"$outputFilePath`" > $buildOutputPath 2>&1"
124+
125+
Invoke-LoggedCommand $command | Out-Null
126+
}
127+
catch {
128+
Write-Host "Failed calculating dependencies for '$TestDependsOnDependency'. Exit code $LASTEXITCODE."
129+
Write-Host "Dumping erroring build output."
130+
Write-Host (Get-Content -Raw $buildOutputPath)
131+
continue
132+
}
133+
134+
if (Test-Path $outputFilePath) {
135+
$dependentProjects = Get-Content $outputFilePath
136+
137+
foreach ($packageRootPath in $dependentProjects) {
138+
if (!$packageRootPath) {
139+
Write-Verbose "Get-dotnet-AdditionalValidationPackagesFromPackageSet::dependentProjects Package root path is empty, skipping."
140+
continue
141+
}
142+
$pkg = $AllPkgProps | Where-Object { $_.DirectoryPath -eq $packageRootPath }
143+
144+
if (!$pkg) {
145+
Write-Verbose "Unable to find package for path $packageRootPath, skipping. Most likely a nested test project not directly under test."
146+
continue
147+
}
148+
149+
if ($pkg -and $LocatedPackages -notcontains $pkg) {
150+
$pkg.IncludedForValidation = $true
151+
$additionalValidationPackages += $pkg
152+
}
153+
}
154+
}
155+
156+
return $additionalValidationPackages
157+
}
158+
169159
# Returns the nuget publish status of a package id and version.
170160
function IsNugetPackageVersionPublished ($pkgId, $pkgVersion)
171161
{

‎eng/service.proj

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -163,13 +163,13 @@
163163
SkipNonexistentTargets="true">
164164
<Output ItemName="ProjectsToInclude" TaskParameter="TargetOutputs"/>
165165
</MSBuild>
166-
<RemoveDuplicates Inputs="@(ProjectsToInclude->Replace($(RepoRoot), ''))">
166+
<RemoveDuplicates Inputs="@(ProjectsToInclude)">
167167
<Output TaskParameter="Filtered" ItemName="ProjectsToIncludeFitered"/>
168168
</RemoveDuplicates>
169169
<Message Text="Write to file $(OutputProjectFilePath)" Importance="high"/>
170-
<Message Text="%24%28RepoRoot%29%(ProjectsToIncludeFitered.Identity)" Importance="high"/>
170+
<Message Text="%(ProjectsToIncludeFitered.Identity)" Importance="high"/>
171171
<ItemGroup>
172-
<_WriteToLines Include="%24%28RepoRoot%29%(ProjectsToIncludeFitered.Identity)" />
172+
<_WriteToLines Include="%(ProjectsToIncludeFitered.Identity)" />
173173
</ItemGroup>
174174
<WriteLinesToFile File="$(OutputProjectFilePath)" Lines="@(_WriteToLines)" />
175175
</Target>

0 commit comments

Comments
 (0)