Skip to content

fix: wrap compiler calls on manylinux_2_34_x86_64 #1727

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mayeut merged 1 commit into from
Dec 15, 2024
Merged

fix: wrap compiler calls on manylinux_2_34_x86_64 #1727

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions README.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,14 @@ for repeatable builds.
manylinux_2_34 (AlmaLinux 9 based)
~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Caveat:
On x86_64, RHEL 9+ derivatives are using x86-64-v2 target architecture.
While manylinux worked around that when building from sources by intercepting compiler calls to target
x86_64 instead, every library installed with dnf will most likely target the more recent x86-64-v2 which, if
grafted into a wheel, will fail to run on older hardware. There's no PEP to handle micro-architecture variants
yet when it comes to packaging or installing wheels. Auditwheel doesn't detect this either.
See https://github.com/pypa/manylinux/issues/1725

Toolchain: GCC 14

- x86_64 image: ``quay.io/pypa/manylinux_2_34_x86_64``
Expand Down
2 changes: 1 addition & 1 deletion build.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ elif [ "${POLICY}" == "manylinux_2_28" ]; then
elif [ "${POLICY}" == "manylinux_2_34" ]; then
BASEIMAGE="almalinux:9"
DEVTOOLSET_ROOTPATH="/opt/rh/gcc-toolset-14/root"
PREPEND_PATH="${DEVTOOLSET_ROOTPATH}/usr/bin:"
PREPEND_PATH="/usr/local/bin:${DEVTOOLSET_ROOTPATH}/usr/bin:"
LD_LIBRARY_PATH_ARG="${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst"
elif [ "${POLICY}" == "musllinux_1_2" ]; then
BASEIMAGE="alpine:3.20"
Expand Down
2 changes: 1 addition & 1 deletion docker/Dockerfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ ARG POLICY=manylinux_2_34
ARG PLATFORM=x86_64
ARG DEVTOOLSET_ROOTPATH=/opt/rh/gcc-toolset-14/root
ARG LD_LIBRARY_PATH_ARG=${DEVTOOLSET_ROOTPATH}/usr/lib64:${DEVTOOLSET_ROOTPATH}/usr/lib:${DEVTOOLSET_ROOTPATH}/usr/lib64/dyninst:${DEVTOOLSET_ROOTPATH}/usr/lib/dyninst
ARG PREPEND_PATH=${DEVTOOLSET_ROOTPATH}/usr/bin:
ARG PREPEND_PATH=/usr/local/bin:${DEVTOOLSET_ROOTPATH}/usr/bin:

FROM $BASEIMAGE AS runtime_base
ARG POLICY
Expand Down
4 changes: 4 additions & 0 deletions docker/build_scripts/build-cpython.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,10 @@ fi

unset _PYTHON_HOST_PLATFORM

if [ "${AUDITWHEEL_ARCH}" == "x86_64" ] && echo | gcc -S -x c -v - 2>&1 | grep 'march=x86-64-v' > /dev/null; then
export EXTRA_CFLAGS="-mtune=generic -march=x86-64"
fi

# configure with hardening options only for the interpreter & stdlib C extensions
# do not change the default for user built extension (yet?)
./configure \
Expand Down
3 changes: 3 additions & 0 deletions docker/build_scripts/finalize.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,3 +98,6 @@ hardlink -c /opt/_internal

# update system packages
LC_ALL=C ${MY_DIR}/update-system-packages.sh

# wrap compilers (see https://github.com/pypa/manylinux/issues/1725)
${MY_DIR}/install-gcc-wrapper.sh
108 changes: 108 additions & 0 deletions docker/build_scripts/install-gcc-wrapper.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
#!/bin/bash

# Stop at any error, show all commands
set -exuo pipefail

if [ "${AUDITWHEEL_ARCH}" != "x86_64" ]; then
exit 0
fi

if ! echo | gcc -S -x c -v - 2>&1 | grep 'march=x86-64-v' > /dev/null; then
exit 0
fi

# Get script directory
MY_DIR=$(dirname "${BASH_SOURCE[0]}")

# Get build utilities
source "${MY_DIR}/build_utils.sh"

# create wrapper to override default -march=x86-64-v? and replace it with -march=x86-64
cat <<EOF > /tmp/manylinux-gcc-wrapper.c
#define _GNU_SOURCE
#include <limits.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>

int main(int argc, char* argv[]) {
int has_march = 0;
int has_mtune = 0;
for (int i = 1; i < argc; ++i) {
if (!has_march && (strncmp(argv[i], "-march=", 7) == 0)) {
has_march = 1;
if (has_mtune) break;
}
else if (!has_mtune && (strncmp(argv[i], "-mtune=", 7) == 0)) {
has_mtune = 1;
if (has_march) break;
}
}
int insert = 0;
if (!has_march) {
insert += 1;
if (!has_mtune) insert += 1;
}
if (argc > (INT_MAX - insert - 1)) {
fputs("too many arguments\n", stderr);
return EXIT_FAILURE;
}
size_t argc_ = argc + insert + 1;
if (argc_ > SIZE_MAX / sizeof(char*)) {
fputs("too many arguments2\n", stderr);
return EXIT_FAILURE;
}
char** argv_ = malloc(argc_ * sizeof(char*));
if (argv_ == NULL) {
fputs("can't allocate memory for arguments\n", stderr);
return EXIT_FAILURE;
}
char* progname = basename(argv[0]);
char argv0[128];
int len = snprintf(argv0, sizeof(argv0), "${DEVTOOLSET_ROOTPATH}/usr/bin/%s", progname);
if ((len <= 0) || (len >= sizeof(argv0))) {
fputs("can't compute argv0\n", stderr);
return EXIT_FAILURE;
}
argv_[0] = argv0;
if (insert > 0) {
if (insert == 2) argv_[1] = "-mtune=generic";
argv_[insert] = "-march=x86-64";
}
for (int i = 1; i < argc; ++i) {
argv_[i + insert] = argv[i];
}
argv_[argc_ - 1] = NULL;
if (execv(argv0, argv_) == -1) {
fprintf(stderr, "failed to start '%s'\n", argv0);
return EXIT_FAILURE;
}
return 0;
}
EOF

gcc ${MANYLINUX_CFLAGS} -std=c11 -Os -s -Werror -o /usr/local/bin/manylinux-gcc-wrapper /tmp/manylinux-gcc-wrapper.c

for EXE in "${DEVTOOLSET_ROOTPATH}"/usr/bin/*; do
if diff -q "${EXE}" "${DEVTOOLSET_ROOTPATH}/usr/bin/gcc"; then
LINK_NAME=/usr/local/bin/$(basename "${EXE}")
ln -s manylinux-gcc-wrapper "${LINK_NAME}"
if echo | "${LINK_NAME}" -S -x c -v - 2>&1 | grep 'march=x86-64-v' > /dev/null; then
exit 1
fi
elif diff -q "${EXE}" "${DEVTOOLSET_ROOTPATH}/usr/bin/g++"; then
LINK_NAME=/usr/local/bin/$(basename "${EXE}")
ln -s manylinux-gcc-wrapper "${LINK_NAME}"
if echo | "${LINK_NAME}" -S -x c++ -v - 2>&1 | grep 'march=x86-64-v' > /dev/null; then
exit 1
fi
elif diff -q "${EXE}" "${DEVTOOLSET_ROOTPATH}/usr/bin/gfortran"; then
LINK_NAME=/usr/local/bin/$(basename "${EXE}")
ln -s manylinux-gcc-wrapper "${LINK_NAME}"
if echo | "${LINK_NAME}" -S -x f77 -v - 2>&1 | grep 'march=x86-64-v' > /dev/null; then
exit 1
fi
fi
done
11 changes: 11 additions & 0 deletions tests/run_tests.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,5 +161,16 @@ fi
# check the default shell is /bin/bash
test "$SHELL" = "/bin/bash"

# https://github.com/pypa/manylinux/issues/1725
# check the compiler does not default to x86-64-v?
if [ "${AUDITWHEEL_ARCH}" == "x86_64" ]; then
which gcc
gcc --version
if echo | gcc -S -x c -v - 2>&1 | grep 'march=x86-64-v'; then
echo "wrong target architecture"
exit 1
fi
fi

# final report
echo "run_tests successful!"
Loading