Skip to content

Commit 0370e8f

Browse files
hunschehunsche
committed
refactor: Standardize Ubuntu image configurations
Aligns ubuntu-20-04 image configurations to match their legacy counterparts, ensuring consistency and simplifying maintenance. - Version-specific scripts (e.g., *_ubuntu_20_04.sh) are updated to mirror the content of their generic legacy equivalents. - ubuntu-20-04 Dockerfiles are modified to replicate the logic of the legacy Dockerfiles. - The LLVM version for ubuntu-24-04 images is reverted to the legacy version to maintain toolchain consistency across all images. Adds a new GitHub Actions workflow (sync-checker.yml) to automatically verify that changes to legacy files are propagated to their Ubuntu version-specific counterparts in future pull requests.
1 parent b80be83 commit 0370e8f

File tree

10 files changed

+132
-29
lines changed

10 files changed

+132
-29
lines changed

‎.github/workflows/sync-checker.yml‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
# Copyright 2023 Google LLC
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
#
15+
################################################################################
16+
17+
name: 'Check Ubuntu Config Sync'
18+
19+
on:
20+
pull_request:
21+
types: [opened, synchronize, reopened]
22+
23+
jobs:
24+
check-sync:
25+
runs-on: ubuntu-latest
26+
steps:
27+
- name: 'Checkout code'
28+
uses: actions/checkout@v4
29+
with:
30+
# Fetch all history so we can diff against the base branch.
31+
fetch-depth: 0
32+
33+
- name: 'Run sync check'
34+
run: |
35+
set -e
36+
37+
MODIFIED_FILES=$(git diff --name-only ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }})
38+
echo "Checking for synchronized file changes..."
39+
echo "Modified files in this PR:"
40+
echo "$MODIFIED_FILES"
41+
42+
ERRORS=""
43+
44+
# Define the mapping of legacy files to their versioned counterparts.
45+
# Format: "legacy_file;versioned_file_pattern"
46+
# The pattern uses {version} which will be replaced with "ubuntu-20-04" and "ubuntu-24-04".
47+
# For Dockerfiles, the pattern is different from scripts.
48+
declare -A LEGACY_DOCKERFILES
49+
LEGACY_DOCKERFILES["infra/base-images/base-builder-fuzzbench/Dockerfile"]="infra/base-images/base-builder-fuzzbench/{version}.Dockerfile"
50+
LEGACY_DOCKERFILES["infra/base-images/base-builder-swift/Dockerfile"]="infra/base-images/base-builder-swift/{version}.Dockerfile"
51+
LEGACY_DOCKERFILES["infra/base-images/base-builder/Dockerfile"]="infra/base-images/base-builder/{version}.Dockerfile"
52+
LEGACY_DOCKERFILES["infra/base-images/base-clang/Dockerfile"]="infra/base-images/base-clang/{version}.Dockerfile"
53+
LEGACY_DOCKERFILES["infra/base-images/base-runner/Dockerfile"]="infra/base-images/base-runner/{version}.Dockerfile"
54+
55+
declare -A LEGACY_SCRIPTS
56+
LEGACY_SCRIPTS["infra/base-images/base-builder-fuzzbench/fuzzbench_install_dependencies"]="infra/base-images/base-builder-fuzzbench/fuzzbench_install_dependencies_{version}"
57+
LEGACY_SCRIPTS["infra/base-images/base-builder/install_swift.sh"]="infra/base-images/base-builder/install_swift_{version}.sh"
58+
LEGACY_SCRIPTS["infra/base-images/base-builder/precompile_honggfuzz"]="infra/base-images/base-builder/precompile_honggfuzz_{version}"
59+
LEGACY_SCRIPTS["infra/base-images/base-clang/checkout_build_install_llvm.sh"]="infra/base-images/base-clang/checkout_build_install_llvm_{version}.sh"
60+
LEGACY_SCRIPTS["infra/base-images/base-runner/install_deps.sh"]="infra/base-images/base-runner/install_deps_{version}.sh"
61+
62+
VERSIONS=("ubuntu-20-04" "ubuntu-24-04")
63+
64+
# Check Dockerfiles
65+
for legacy_file in "${{!LEGACY_DOCKERFILES[@]}}"; do
66+
if echo "$MODIFIED_FILES" | grep -q "^${legacy_file}$"; then
67+
echo "Legacy file changed: $legacy_file. Verifying counterparts..."
68+
for version in "${{VERSIONS[@]}}"; do
69+
pattern=${{LEGACY_DOCKERFILES[$legacy_file]}}
70+
versioned_file="${{pattern/{{version}}/$version}}"
71+
if ! echo "$MODIFIED_FILES" | grep -q "^${{versioned_file}}$"; then
72+
ERRORS+="\n- Legacy file '${legacy_file}' was changed, but its counterpart '${versioned_file}' was not."
73+
fi
74+
done
75+
fi
76+
done
77+
78+
# Check Scripts
79+
for legacy_file in "${{!LEGACY_SCRIPTS[@]}}"; do
80+
if echo "$MODIFIED_FILES" | grep -q "^${legacy_file}$"; then
81+
echo "Legacy script changed: $legacy_file. Verifying counterparts..."
82+
for version in "${{VERSIONS[@]}}"; do
83+
pattern=${{LEGACY_SCRIPTS[$legacy_file]}}
84+
versioned_file="${{pattern/{{version}}/$version}}"
85+
if ! echo "$MODIFIED_FILES" | grep -q "^${{versioned_file}}$"; then
86+
ERRORS+="\n- Legacy script '${legacy_file}' was changed, but its counterpart '${versioned_file}' was not."
87+
fi
88+
done
89+
fi
90+
done
91+
92+
if [ -n "$ERRORS" ]; then
93+
echo -e "\n\e[31mError: Found synchronization issues between legacy and versioned files.\e[0m"
94+
echo -e "Please update the following files to match their legacy counterparts or ensure they are included in this PR:$ERRORS"
95+
exit 1
96+
else
97+
echo -e "\n\e[32mSuccess: All modified legacy files are synchronized with their versioned counterparts.\e[0m"
98+
fi

‎infra/base-images/base-builder-fuzzbench/fuzzbench_install_dependencies_ubuntu_20_04‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,4 +19,4 @@ apt-get update && apt-get install -y gcc gfortran python-dev libopenblas-dev lib
1919
wget -O /tmp/requirements.txt https://raw.githubusercontent.com/google/fuzzbench/master/requirements.txt
2020
pip3 install pip --upgrade
2121
CFLAGS= CXXFLAGS= pip3 install -r /tmp/requirements.txt
22-
rm /tmp/requirements.txt
22+
rm /tmp/requirements.txt

‎infra/base-images/base-builder-swift/ubuntu-20-04.Dockerfile‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616

1717
FROM gcr.io/oss-fuzz-base/base-builder:ubuntu-20-04
1818

19-
COPY llvmsymbol.diff /src/
2019
RUN install_swift_ubuntu_20_04.sh
2120

2221
COPY precompile_swift /usr/local/bin/

‎infra/base-images/base-builder/install_swift_ubuntu_20_04.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash -eux
2-
# Copyright 2025 Google LLC
2+
# Copyright 2021 Google LLC
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -64,4 +64,4 @@ rm -rf llvm-project llvmsymbol.diff
6464

6565
# TODO: Cleanup packages
6666
apt-get remove --purge -y wget zlib1g-dev
67-
apt-get autoremove -y
67+
apt-get autoremove -y

‎infra/base-images/base-builder/precompile_honggfuzz_ubuntu_20_04‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
#!/bin/bash -eu
1+
#!/bin/bash -eux
22
# Copyright 2019 Google Inc.
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
@@ -33,7 +33,8 @@ make clean
3333
# These CFLAGs match honggfuzz's default, with the exception of -mtune to
3434
# improve portability and `-D_HF_LINUX_NO_BFD` to remove assembly instructions
3535
# from the filenames.
36-
CC=clang CFLAGS="-O3 -funroll-loops -D_HF_LINUX_NO_BFD" make
36+
sed -i 's/-Werror//g' Makefile
37+
CC=clang CFLAGS="-O3 -funroll-loops -D_HF_LINUX_NO_BFD -Wno-unterminated-string-initialization -Wno-error" make
3738

3839
# libhfuzz.a will be added by CC/CXX linker directly during linking,
3940
# but it's defined here to satisfy the build infrastructure
@@ -42,4 +43,4 @@ popd > /dev/null
4243

4344
apt-get remove -y --purge ${PACKAGES[@]}
4445
apt-get autoremove -y
45-
echo "Done."
46+
echo "Done."

‎infra/base-images/base-builder/ubuntu-20-04.Dockerfile‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,10 +122,12 @@ ENV FUZZER_LDFLAGS ""
122122

123123
WORKDIR $SRC
124124

125+
COPY afl_llvm22_patch.diff $SRC/
125126
RUN git clone https://github.com/AFLplusplus/AFLplusplus.git aflplusplus && \
126127
cd aflplusplus && \
127-
git checkout daaefcddc063b356018c29027494a00bcfc3e240 && \
128+
git checkout eadc8a2a7e0fa0338802ee6254bf296489ce4fd7 && \
128129
wget --no-check-certificate -O oss.sh https://raw.githubusercontent.com/vanhauser-thc/binary_blobs/master/oss.sh && \
130+
git apply $SRC/afl_llvm22_patch.diff && \
129131
rm -rf .git && \
130132
chmod 755 oss.sh
131133

‎infra/base-images/base-clang/checkout_build_install_llvm_ubuntu_20_04.sh‎

Lines changed: 19 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash -eux
2-
# Copyright 2025 Google LLC
2+
# Copyright 2016 Google Inc.
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -60,14 +60,15 @@ apt-get update && apt-get install -y $LLVM_DEP_PACKAGES --no-install-recommends
6060
# languages, projects, ...) is needed.
6161
# Check CMAKE_VERSION infra/base-images/base-clang/Dockerfile was released
6262
# recently enough to fully support this clang version.
63-
OUR_LLVM_REVISION=llvmorg-18.1.8
63+
OUR_LLVM_REVISION=cb2f0d0a5f14
6464

6565
mkdir $SRC/chromium_tools
6666
cd $SRC/chromium_tools
6767
git clone https://chromium.googlesource.com/chromium/src/tools/clang
6868
cd clang
6969
# Pin clang script due to https://github.com/google/oss-fuzz/issues/7617
70-
git checkout 9eb79319239629c1b23cf7a59e5ebb2bab319a34
70+
OUR_CLANG_REVISION=063d3766486a820c708e888d737b004d11543410
71+
git checkout $OUR_CLANG_REVISION
7172

7273
LLVM_SRC=$SRC/llvm-project
7374
# Checkout
@@ -79,7 +80,7 @@ function clone_with_retries {
7980

8081
# Disable exit on error since we might encounter some failures while retrying.
8182
set +e
82-
for i in $(seq 1 $CHECKOUT_RETRIES); do
83+
for i in $(seq 1 $CHECKOUT_RETRIES);
8384
rm -rf $LOCAL_PATH
8485
git clone $REPOSITORY $LOCAL_PATH
8586
CHECKOUT_RETURN_CODE=$?
@@ -97,7 +98,16 @@ clone_with_retries https://github.com/llvm/llvm-project.git $LLVM_SRC
9798
git -C $LLVM_SRC checkout $OUR_LLVM_REVISION
9899
echo "Using LLVM revision: $OUR_LLVM_REVISION"
99100

100-
# For fuzz introspector.
101+
# Prepare fuzz introspector.
102+
echo "Installing fuzz introspector"
103+
FUZZ_INTROSPECTOR_CHECKOUT=341ebbd72bc9116733bcfcfab5adfd7f9b633e07
104+
105+
git clone https://github.com/ossf/fuzz-introspector.git /fuzz-introspector
106+
cd /fuzz-introspector
107+
git checkout $FUZZ_INTROSPECTOR_CHECKOUT
108+
git submodule init
109+
git submodule update
110+
101111
echo "Applying introspector changes"
102112
OLD_WORKING_DIR=$PWD
103113
cd $LLVM_SRC
@@ -107,6 +117,7 @@ cp -rf /fuzz-introspector/frontends/llvm/lib/Transforms/FuzzIntrospector ./llvm/
107117
# LLVM currently does not support dynamically loading LTO passes. Thus, we
108118
# hardcode it into Clang instead. Ref: https://reviews.llvm.org/D77704
109119
/fuzz-introspector/frontends/llvm/patch-llvm.sh
120+
110121
cd $OLD_WORKING_DIR
111122

112123
mkdir -p $WORK/llvm-stage2 $WORK/llvm-stage1
@@ -134,7 +145,7 @@ if [[ -n "$FULL_LLVM_BUILD" ]]; then
134145
export LIBRARY_PATH=/usr/local/lib/x86_64-unknown-linux-gnu/
135146
fi
136147

137-
# Note: LLVM_ENABLE_LIBCXX=ON doesn't break the build even if libcxx doesn't
148+
# Note: LLVM_ENABLE_LIBCXX=ON doesn't break the build even if libcxx doesn\'t
138149
# exist.
139150
cmake -G "Ninja" \
140151
-DLIBCXX_ENABLE_SHARED=OFF \
@@ -179,6 +190,7 @@ function free_disk_space {
179190
/usr/local/bin/llvm-as \
180191
/usr/local/bin/llvm-config \
181192
/usr/local/bin/llvm-cov \
193+
/usr/local/bin/llvm-link \
182194
/usr/local/bin/llvm-objcopy \
183195
/usr/local/bin/llvm-nm \
184196
/usr/local/bin/llvm-profdata \
@@ -281,4 +293,4 @@ ninja -j $NPROC cxx
281293
ninja install-cxx
282294
rm -rf $WORK/msan
283295

284-
free_disk_space
296+
free_disk_space

‎infra/base-images/base-clang/checkout_build_install_llvm_ubuntu_24_04.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,14 +60,14 @@ apt-get update && apt-get install -y $LLVM_DEP_PACKAGES --no-install-recommends
6060
# languages, projects, ...) is needed.
6161
# Check CMAKE_VERSION infra/base-images/base-clang/Dockerfile was released
6262
# recently enough to fully support this clang version.
63-
OUR_LLVM_REVISION=llvmorg-18.1.8
63+
OUR_LLVM_REVISION=cb2f0d0a5f14
6464

6565
mkdir $SRC/chromium_tools
6666
cd $SRC/chromium_tools
6767
git clone https://chromium.googlesource.com/chromium/src/tools/clang
6868
cd clang
6969
# Pin clang script due to https://github.com/google/oss-fuzz/issues/7617
70-
git checkout 9eb79319239629c1b23cf7a59e5ebb2bab319a34
70+
git checkout 063d3766486a820c708e888d737b004d11543410
7171

7272
LLVM_SRC=$SRC/llvm-project
7373
# Checkout

‎infra/base-images/base-clang/ubuntu-20-04.Dockerfile‎

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -33,15 +33,6 @@ RUN apt-get update && apt-get install -y wget sudo && \
3333
SUDO_FORCE_REMOVE=yes apt-get autoremove --purge -y wget sudo && \
3434
rm -rf /usr/local/doc/cmake /usr/local/bin/cmake-gui
3535

36-
RUN apt-get update && apt-get install -y git && \
37-
git clone https://github.com/ossf/fuzz-introspector.git fuzz-introspector && \
38-
cd fuzz-introspector && \
39-
git checkout 332d674f00b8abc4c9ebf10e9c42e5b72b331c63 && \
40-
git submodule init && \
41-
git submodule update && \
42-
apt-get autoremove --purge -y git && \
43-
rm -rf .git
44-
4536
COPY checkout_build_install_llvm_ubuntu_20_04.sh /root/
4637
# Keep all steps in the same script to decrease the number of intermediate
4738
# layes in docker file.
@@ -67,12 +58,12 @@ ENV CCC "clang++"
6758
ENV CFLAGS -O1 \
6859
-fno-omit-frame-pointer \
6960
-gline-tables-only \
70-
-Wno-error=enum-constexpr-conversion \
7161
-Wno-error=incompatible-function-pointer-types \
7262
-Wno-error=int-conversion \
7363
-Wno-error=deprecated-declarations \
7464
-Wno-error=implicit-function-declaration \
7565
-Wno-error=implicit-int \
66+
-Wno-error=unknown-warning-option \
7667
-Wno-error=vla-cxx-extension \
7768
-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
7869
ENV CXXFLAGS_EXTRA "-stdlib=libc++"

‎infra/base-images/base-runner/install_deps_ubuntu_20_04.sh‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/bin/bash -eux
2-
# Copyright 2025 Google LLC
2+
# Copyright 2022 Google LLC
33
#
44
# Licensed under the Apache License, Version 2.0 (the "License");
55
# you may not use this file except in compliance with the License.
@@ -35,4 +35,4 @@ case $(uname -m) in
3535
# We only need to worry about i386 if we are on x86_64.
3636
apt-get install -y lib32gcc1 libc6-i386
3737
;;
38-
esac
38+
esac

0 commit comments

Comments
 (0)