Skip to content

Commit 1ec5992

Browse files
JkintalixJkintalix
committed
Merge branch '3.0-dev' into fix-docker-startup-overlay2
Signed-off-by: kinatli jayanth <jayanthx.kintali@intel.com>
2 parents 7b082d9 + 18d4421 commit 1ec5992

File tree

965 files changed

+94113
-28238
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

965 files changed

+94113
-28238
lines changed

‎.github/actions/checkout-with-stable-pkgs/action.yml‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
# Copyright (c) Microsoft Corporation.
2+
# Licensed under the MIT License.
3+
4+
name: "Stable specs and manifests checkout"
5+
description: "Checks out the repo, and a stable version of both specs and manifests."
6+
runs:
7+
using: "composite"
8+
steps:
9+
- name: Checkout repository
10+
uses: actions/checkout@v4
11+
with:
12+
fetch-depth: 0
13+
fetch-tags: true
14+
15+
- name: Checkout stable specs and manifests
16+
shell: bash
17+
run: git checkout origin/3.0 -- SPECS/ toolkit/resources/manifests/package/*.txt

‎.github/workflows/check-files.yml‎

Lines changed: 140 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,140 @@
1+
# Copyright (c) Microsoft Corporation.
2+
# Licensed under the MIT License.
3+
4+
name: Check Disallowed Files
5+
6+
on:
7+
push:
8+
branches: [main, 2.0*, 3.0*, fasttrack/*]
9+
pull_request:
10+
branches: [main, 2.0*, 3.0*, fasttrack/*]
11+
12+
jobs:
13+
14+
build:
15+
name: Check Disallowed Files
16+
runs-on: ubuntu-latest
17+
steps:
18+
19+
- name: Check out code
20+
uses: actions/checkout@v4
21+
22+
- name: Get base commit for PRs
23+
if: ${{ github.event_name == 'pull_request' }}
24+
run: |
25+
git fetch origin ${{ github.base_ref }}
26+
echo "base_sha=$(git rev-parse origin/${{ github.base_ref }})" >> $GITHUB_ENV
27+
echo "Merging ${{ github.sha }} into ${{ github.base_ref }}"
28+
29+
- name: Get base commit for Pushes
30+
if: ${{ github.event_name == 'push' }}
31+
run: |
32+
git fetch origin ${{ github.event.before }}
33+
echo "base_sha=${{ github.event.before }}" >> $GITHUB_ENV
34+
echo "Merging ${{ github.sha }} into ${{ github.event.before }}"
35+
36+
- name: Get the changed files
37+
run: |
38+
echo "Files changed: '$(git diff-tree --no-commit-id --name-only -r ${{ env.base_sha }} ${{ github.sha }})'"
39+
changed_files=$(git diff-tree --diff-filter=AM --no-commit-id --name-only -r ${{ env.base_sha }} ${{ github.sha }})
40+
echo "Files to validate: '${changed_files}'"
41+
echo "changed-files<<EOF" >> $GITHUB_ENV
42+
echo "${changed_files}" >> $GITHUB_ENV
43+
echo "EOF" >> $GITHUB_ENV
44+
45+
- name: Check for disallowed file types
46+
run: |
47+
if [[ -z "${{ env.changed-files }}" ]]; then
48+
echo "No files to validate. Exiting."
49+
exit 0
50+
fi
51+
52+
echo "Checking files..."
53+
error_found=0
54+
55+
# Read disallowed extensions from the configuration file
56+
if [[ ! -f ".github/workflows/disallowed-extensions.txt" ]]; then
57+
echo "Configuration file '.github/workflows/disallowed-extensions.txt' not found. Skipping check."
58+
exit 0
59+
fi
60+
61+
# Create array of disallowed extensions
62+
mapfile -t disallowed_extensions < .github/workflows/disallowed-extensions.txt
63+
if [[ $? -ne 0 ]]; then
64+
echo "Error occurred while reading disallowed extensions. Exiting."
65+
exit 1
66+
fi
67+
68+
# Check each changed file
69+
while IFS= read -r file; do
70+
if [[ -z "$file" ]]; then
71+
continue
72+
fi
73+
74+
echo "Checking file: $file"
75+
76+
# Get file extension (convert to lowercase for comparison)
77+
extension=$(echo "${file##*.}" | tr '[:upper:]' '[:lower:]')
78+
filename=$(basename "$file")
79+
80+
# Check if file should be in blob store
81+
should_be_in_blob_store=false
82+
83+
# Check against disallowed extensions
84+
for disallowed_ext in "${disallowed_extensions[@]}"; do
85+
# Remove any whitespace and comments
86+
clean_ext=$(echo "$disallowed_ext" | sed 's/#.*//' | xargs)
87+
if [[ -z "$clean_ext" ]]; then
88+
continue
89+
fi
90+
91+
if [[ "$extension" == "$clean_ext" ]]; then
92+
should_be_in_blob_store=true
93+
break
94+
fi
95+
done
96+
97+
# Additional checks for binary files and large files
98+
if [[ -f "$file" ]]; then
99+
# Check if file is binary (but allow .sh files even if executable)
100+
if file "$file" | grep -q "binary\|archive\|compressed"; then
101+
should_be_in_blob_store=true
102+
fi
103+
104+
# Check file size (files > 1MB should be in blob store)
105+
file_size=$(stat -f%z "$file" 2>/dev/null || stat -c%s "$file" 2>/dev/null || echo 0)
106+
if [[ $file_size -gt 1048576 ]]; then # 1MB
107+
should_be_in_blob_store=true
108+
fi
109+
fi
110+
111+
if [[ "$should_be_in_blob_store" == "true" ]]; then
112+
1>&2 echo "**** ERROR ****"
113+
1>&2 echo "File '$file' should be stored in blob store, not in git repository."
114+
1>&2 echo "Reason: Images, Large files, binaries, tarballs, and non-text files slow down git operations"
115+
1>&2 echo "and cannot be efficiently diffed. Please upload to blob store instead."
116+
1>&2 echo "**** ERROR ****"
117+
error_found=1
118+
fi
119+
done <<< "${{ env.changed-files }}"
120+
121+
if [[ $error_found -eq 1 ]]; then
122+
echo ""
123+
echo "=========================================="
124+
echo "FILES THAT SHOULD BE IN BLOB STORE DETECTED"
125+
echo "=========================================="
126+
echo "The following file types should be stored in blob store:"
127+
echo "- Source tarballs (.tar.gz, .tar.xz, .zip, etc.)"
128+
echo "- Binary files (.bin, .exe, .so, .dll, etc.)"
129+
echo "- Images (.gif, .bmp, etc.)"
130+
echo "- Archives (.rar, .7z, .tar, etc.)"
131+
echo "- Large files (> 1MB)"
132+
echo "- Any non-text files that cannot be efficiently diffed"
133+
echo ""
134+
echo "Please upload these files to the blob store and reference them"
135+
echo "in your spec files or configuration instead of checking them into git."
136+
echo "=========================================="
137+
exit 1
138+
fi
139+
140+
echo "All files are appropriate for git storage."

‎.github/workflows/check-package-builds.yml‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
# Copyright (c) Microsoft Corporation.
2+
# Licensed under the MIT License.
3+
4+
# This check verifies basic package build success and failure cases.
5+
# It should only be dependent on toolkit changes, not on the specs.
6+
# This is why each build uses the 3.0-stable version of the specs and manifests.
7+
8+
name: Package build checks
9+
10+
env:
11+
REGULAR_PKG: words
12+
REGULAR_PKG_SPEC_PATH: SPECS/words/words.spec
13+
TOOLCHAIN_PKG: xz
14+
15+
on:
16+
push:
17+
branches: [3.0*, fasttrack/3.0]
18+
paths:
19+
- ".github/workflows/check-package-builds.yml"
20+
- "toolkit/Makefile"
21+
- "toolkit/scripts/*"
22+
- "toolkit/tools/*"
23+
pull_request:
24+
branches: [3.0*, fasttrack/3.0]
25+
paths:
26+
- ".github/workflows/check-package-builds.yml"
27+
- "toolkit/Makefile"
28+
- "toolkit/scripts/*"
29+
- "toolkit/tools/*"
30+
31+
jobs:
32+
package-checks:
33+
name: ${{ matrix.check-name }}
34+
runs-on: ubuntu-latest
35+
strategy:
36+
fail-fast: false
37+
matrix:
38+
include:
39+
- check-name: "Simple package build succeeds"
40+
package-type: "REGULAR_PKG"
41+
extra-args: ""
42+
43+
- check-name: "Simple package build fails"
44+
package-type: "REGULAR_PKG"
45+
error-pattern: "Number of failed SRPMs:\\s+1\\s*$"
46+
extra-args: ""
47+
build-prep: |
48+
# Adding an invalid command to the '%prep' section will cause the build to fail.
49+
sed -i '/%prep/a this-command-should-fail-because-its-not-a-command-at-all' "$REGULAR_PKG_SPEC_PATH"
50+
51+
- check-name: "Toolchain package rebuild succeeds"
52+
package-type: "TOOLCHAIN_PKG"
53+
extra-args: "ALLOW_TOOLCHAIN_REBUILDS=y"
54+
55+
- check-name: "Toolchain package rebuild fails"
56+
package-type: "TOOLCHAIN_PKG"
57+
error-pattern: "Number of toolchain SRPM conflicts:\\s+1\\s*$"
58+
extra-args: "ALLOW_TOOLCHAIN_REBUILDS=n"
59+
build-prep: ""
60+
61+
- check-name: "None license check does not break the build"
62+
package-type: "REGULAR_PKG"
63+
extra-args: "LICENSE_CHECK_MODE=none"
64+
build-prep: |
65+
license_file_name=$(grep -oP '^%license\s+\K\S+' "$REGULAR_PKG_SPEC_PATH")
66+
if [[ -z "$license_file_name" ]]; then
67+
echo "ERROR: no license file found in the spec $REGULAR_PKG_SPEC_PATH"
68+
exit 1
69+
fi
70+
# Tagging a license file as a documentation file will not fail the license check on the 'none' level.
71+
sed -i "/^%license/a %doc $license_file_name" "$REGULAR_PKG_SPEC_PATH"
72+
73+
- check-name: "Warning-only license check does not break the build"
74+
package-type: "REGULAR_PKG"
75+
extra-args: "LICENSE_CHECK_MODE=warn"
76+
build-prep: |
77+
license_file_name=$(grep -oP '^%license\s+\K\S+' "$REGULAR_PKG_SPEC_PATH")
78+
if [[ -z "$license_file_name" ]]; then
79+
echo "ERROR: no license file found in the spec $REGULAR_PKG_SPEC_PATH"
80+
exit 1
81+
fi
82+
# Tagging a license file as a documentation file will not fail the license check on the 'warn' level.
83+
sed -i "/^%license/a %doc $license_file_name" "$REGULAR_PKG_SPEC_PATH"
84+
85+
- check-name: "Fatal license check succeeds on duplicated license as documentation"
86+
package-type: "REGULAR_PKG"
87+
extra-args: "LICENSE_CHECK_MODE=fatal"
88+
build-prep: |
89+
license_file_name=$(grep -oP '^%license\s+\K\S+' "$REGULAR_PKG_SPEC_PATH")
90+
if [[ -z "$license_file_name" ]]; then
91+
echo "ERROR: no license file found in the spec $REGULAR_PKG_SPEC_PATH"
92+
exit 1
93+
fi
94+
# Tagging a license file as a documentation file will not fail the license check on the 'fatal' level.
95+
sed -i "/^%license/a %doc $license_file_name" "$REGULAR_PKG_SPEC_PATH"
96+
97+
- check-name: "Fatal license check fails"
98+
package-type: "REGULAR_PKG"
99+
error-pattern: "Number of SRPMs with license errors:\\s+1\\s*$"
100+
extra-args: "LICENSE_CHECK_MODE=fatal"
101+
build-prep: |
102+
if ! grep -q '^%license' "$REGULAR_PKG_SPEC_PATH"; then
103+
echo "ERROR: no '%license' macro found in the spec $REGULAR_PKG_SPEC_PATH"
104+
exit 1
105+
fi
106+
# Tagging a license file as a documentation file will cause the license check to fail.
107+
sed -i "s/^%license/%doc/" "$REGULAR_PKG_SPEC_PATH"
108+
109+
- check-name: "Pedantic license check fails"
110+
package-type: "REGULAR_PKG"
111+
error-pattern: "Number of SRPMs with license errors:\\s+1\\s*$"
112+
extra-args: "LICENSE_CHECK_MODE=pedantic"
113+
build-prep: |
114+
license_file_name=$(grep -oP '^%license\s+\K\S+' "$REGULAR_PKG_SPEC_PATH")
115+
if [[ -z "$license_file_name" ]]; then
116+
echo "ERROR: no license file found in the spec $REGULAR_PKG_SPEC_PATH"
117+
exit 1
118+
fi
119+
sed -i "/^%license/a %doc $license_file_name" "$REGULAR_PKG_SPEC_PATH"
120+
121+
steps:
122+
- uses: actions/checkout@v4
123+
124+
- name: Checkout a stable version of the specs
125+
uses: ./.github/actions/checkout-with-stable-pkgs
126+
127+
- name: Prepare the build environment
128+
if: ${{ matrix.build-prep != '' }}
129+
run: |
130+
set -euo pipefail
131+
132+
${{ matrix.build-prep }}
133+
134+
- name: Run the build
135+
run: |
136+
set -euo pipefail
137+
138+
if sudo make -C toolkit -j$(nproc) build-packages \
139+
PACKAGE_REBUILD_LIST="${{ env[matrix.package-type] }}" \
140+
REBUILD_TOOLS=y \
141+
SRPM_PACK_LIST="${{ env[matrix.package-type] }}" \
142+
${{ matrix.extra-args }} 2>&1 | tee build.log; then
143+
touch build.succeeded
144+
fi
145+
146+
- name: Check the results
147+
run: |
148+
set -euo pipefail
149+
150+
if [[ -z "${{ matrix.error-pattern }}" ]]; then
151+
if [[ ! -f build.succeeded ]]; then
152+
echo "Build failed, but it was expected to succeed."
153+
exit 1
154+
fi
155+
else
156+
if [[ -f build.succeeded ]]; then
157+
echo "Build succeeded, but it was expected to fail."
158+
exit 1
159+
fi
160+
161+
if ! grep -qP '${{ matrix.error-pattern }}' build.log; then
162+
echo "Build failed, but not with the expected error message."
163+
exit 1
164+
fi
165+
fi

‎.github/workflows/check-package-cgmanifest.yml‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,10 @@ jobs:
5656
echo "Files to validate: '${changed_specs}'"
5757
echo "updated-specs=${changed_specs}" >> "$GITHUB_ENV"
5858
59+
- name: Build the worker chroot
60+
if: ${{ env.updated-specs != '' }}
61+
run: sudo make -C toolkit -j$(nproc) chroot-tools REBUILD_TOOLS=y DAILY_BUILD_ID=lkg
62+
5963
- name: Check each spec
60-
run: |
61-
.github/workflows/overwrite_shell_link.sh
62-
.github/workflows/validate-cg-manifest.sh ${{ env.updated-specs }}
63-
shell: bash
64+
if: ${{ env.updated-specs != '' }}
65+
run: .github/workflows/validate-cg-manifest.sh build/worker/worker_chroot.tar.gz ${{ env.updated-specs }}

0 commit comments

Comments
 (0)