From 3a4335d04251ba3a35e5ee70ffb6ecc6aea03fa5 Mon Sep 17 00:00:00 2001 From: Rok Mandeljc Date: Fri, 29 Jan 2021 21:59:49 +0100 Subject: [PATCH] Add scripts for extracting SDKs from Command Line Tools for Xcode Being able to extract SDKs from Command Line Tools for Xcode instead of the full Xcode package saves bandwidth, disk space, and time. Two scripts are added: * tools/gen_sdk_package_tools.sh: extracts SDKs from installed Command Line Tools on macOS (/Library/Developer/CommandLineTools) or from a path specified via XCODE_TOOLS_DIR * tools/gen_sdk_package_tools_dmg.sh: unpacks the content of provided Command_Line_Tools_for_Xcode.dmg and extracts SDKs via the above script. Tested with Command Line Tools for Xcode 12.x. --- README.md | 33 +++++- tools/gen_sdk_package_tools.sh | 164 +++++++++++++++++++++++++++++ tools/gen_sdk_package_tools_dmg.sh | 104 ++++++++++++++++++ 3 files changed, 300 insertions(+), 1 deletion(-) create mode 100755 tools/gen_sdk_package_tools.sh create mode 100755 tools/gen_sdk_package_tools_dmg.sh diff --git a/README.md b/README.md index f856831..e77d70d 100644 --- a/README.md +++ b/README.md @@ -159,7 +159,10 @@ use these variants unless you know what you're doing. **[Please ensure you have read and understood the Xcode license terms before continuing.](https://www.apple.com/legal/sla/docs/xcode.pdf)** -##### Packaging the SDK on macOS: ##### +The SDKs can be extracted either from full Xcode or from Command Line +Tools for Xcode. + +##### Packaging the SDK on macOS (Xcode): ##### 1. [Download Xcode: https://developer.apple.com/download/more] \*\* 2. [Mount Xcode.dmg (Open With -> DiskImageMounter) \*\*\*] @@ -212,6 +215,34 @@ An SSD is recommended for this method. 6. Copy or move the SDK into the tarballs/ directory +##### Packaging the SDK from Xcode Command Line Tools on macOS: ##### + +1. [Download Xcode Command Line Tools: https://developer.apple.com/download/more] \*\*\*\* +2. [Mount Command_Line_Tools_for_Xcode.dmg (Open With -> DiskImageMounter)] +3. [Install "Command Line Tools.pkg" (Open With -> Installer)] +3. Run: `./tools/gen_sdk_package_tools.sh` (from the OSXCross package) +4. Copy the packaged SDK (\*.tar.\* or \*.pkg) on a USB Stick +5. (On Linux/BSD) Copy or move the SDK into the tarballs/ directory of + OSXCross. + +\*\*\*\* +-- Xcode command line tools 12.x are known to work. + +Steps 1. to 3. can be skipped if you have Xcode Command line tools +already installed (e.g., auto-installed by running `git` or `gcc` +command from command-line). + +##### Packing the SDK from from Xcode Command Line Tools on Linux: ##### + +This method may require up to 25 GB of free disk space. +An SSD is recommended for this method. + +1. Download Xcode Command Line Tools like described in 'Packaging the SDK from Xcode Command Line Tools on macOS' +2. Install `clang`, `make`, `libssl-devel`, `lzma-devel` and `libxml2-devel` +3. Run `./tools/gen_sdk_package_tools_dmg.sh .dmg` +4. Copy or move the SDK into the tarballs/ directory + + ### USAGE EXAMPLES: ### ##### Example. To compile a file called test.cpp, you can run: ##### diff --git a/tools/gen_sdk_package_tools.sh b/tools/gen_sdk_package_tools.sh new file mode 100755 index 0000000..7f1c427 --- /dev/null +++ b/tools/gen_sdk_package_tools.sh @@ -0,0 +1,164 @@ +#!/usr/bin/env bash +# +# Package the macOS SDKs into a tar file to be used by `build.sh`. +# As opposed to gen_sdk_package.sh, which is used for extraction of SDKs +# from full Xcode version, gen_sdk_tools.sh extracts SDKs from Xcode +# Command Line Tools. +# +# Tested with XCode Command Line Tools 12.x +# + +export LC_ALL=C + + +command -v gnutar &>/dev/null + +if [ $? -eq 0 ]; then + TAR=gnutar +else + TAR=tar +fi + + +if [ -z "$SDK_COMPRESSOR" ]; then + command -v xz &>/dev/null + + if [ $? -eq 0 ]; then + SDK_COMPRESSOR=xz + SDK_EXT="tar.xz" + else + SDK_COMPRESSOR=bzip2 + SDK_EXT="tar.bz2" + fi +fi + +case $SDK_COMPRESSOR in + "gz") + SDK_COMPRESSOR=gzip + SDK_EXT=".tar.gz" + ;; + "bzip2") + SDK_EXT=".tar.bz2" + ;; + "xz") + SDK_EXT=".tar.xz" + ;; + "zip") + SDK_EXT=".zip" + ;; + *) + echo "error: unknown compressor \"$SDK_COMPRESSOR\"" >&2 + exit 1 +esac + +function compress() +{ + case $SDK_COMPRESSOR in + "zip") + $SDK_COMPRESSOR -q -5 -r - $1 > $2 ;; + *) + tar cf - $1 | $SDK_COMPRESSOR -5 - > $2 ;; + esac +} + + +function rreadlink() +{ + if [ ! -h "$1" ]; then + echo "$1" + else + local link="$(expr "$(command ls -ld -- "$1")" : '.*-> \(.*\)$')" + cd $(dirname $1) + rreadlink "$link" | sed "s|^\([^/].*\)\$|$(dirname $1)/\1|" + fi +} + + +if [ $(uname -s) != "Darwin" ]; then + if [ -z "$XCODE_TOOLS_DIR" ]; then + echo "This script must be run on macOS" 1>&2 + echo "... Or with XCODE_TOOLS_DIR=... on Linux" 1>&2 + exit 1 + else + case "$XCODE_TOOLS_DIR" in + /*) ;; + *) XCODE_TOOLS_DIR="$PWD/$XCODE_TOOLS_DIR" ;; + esac + fi +else + XCODE_TOOLS_DIR="/Library/Developer/CommandLineTools" +fi + +if [ ! -d "$XCODE_TOOLS_DIR" ]; then + echo "cannot find Xcode Command Line Tools (XCODE_TOOLS_DIR=$XCODE_TOOLS_DIR)" 1>&2 + exit 1 +fi + +echo -e "found Xcode Command Line Tools: $XCODE_TOOLS_DIR" + +WDIR=$(pwd) + +set -e + +pushd "$XCODE_TOOLS_DIR" &>/dev/null + +if [ -d "SDKs" ]; then + pushd "SDKs" &>/dev/null +else + echo "$XCODE_TOOLS_DIR/SDKs does not exist" 1>&2 + exit 1 +fi + +SDKS=$(ls | grep -E "^MacOSX11.*|^MacOSX10.*" | grep -v "Patch") + +if [ -z "$SDKS" ]; then + echo "No SDK found" 1>&2 + exit 1 +fi + +# libc++ headers for C++11/C++14 +LIBCXXDIR="usr/include/c++/v1" + +# Manual directory +MANDIR="usr/share/man" + +for SDK in $SDKS; do + echo -n "packaging $(echo "$SDK" | sed -E "s/(.sdk|.pkg)//g") SDK " + echo "(this may take several minutes) ..." + + if [[ $SDK == *.pkg ]]; then + cp $SDK $WDIR + continue + fi + + TMP=$(mktemp -d /tmp/XXXXXXXXXXX) + cp -r $(rreadlink $SDK) $TMP/$SDK &>/dev/null || true + + pushd "$XCODE_TOOLS_DIR" &>/dev/null + + mkdir -p $TMP/$SDK/usr/include/c++ + + # libc++ headers for C++11/C++14 + if [ -d $LIBCXXDIR ]; then + cp -rf $LIBCXXDIR "$TMP/$SDK/usr/include/c++" + fi + + if [ -d $MANDIR ]; then + mkdir -p $TMP/$SDK/usr/share/man + cp -rf $MANDIR/* $TMP/$SDK/usr/share/man + fi + + popd &>/dev/null + + pushd $TMP &>/dev/null + compress "*" "$WDIR/$SDK$SDK_EXT" + popd &>/dev/null + + rm -rf $TMP +done + +popd &>/dev/null +popd &>/dev/null + +echo "" +ls -lh | grep MacOSX diff --git a/tools/gen_sdk_package_tools_dmg.sh b/tools/gen_sdk_package_tools_dmg.sh new file mode 100755 index 0000000..1777dfd --- /dev/null +++ b/tools/gen_sdk_package_tools_dmg.sh @@ -0,0 +1,104 @@ +#!/usr/bin/env bash +# +# Extract Xcode Command Line Tools .dmg, process "Command Line Tools.pkg" +# and its sub-archives/packages, and run gen_sdk_package_tools.sh. +# +# Tested with XCode Command Line Tools 12.x +# + +pushd "${0%/*}/.." &>/dev/null +source tools/tools.sh + +require cpio + +if [ $PLATFORM == "Darwin" ]; then + echo "Use gen_sdk_package_tools.sh on macOS" 1>&2 + exit 1 +fi + +if [ $# -eq 0 ]; then + echo "Usage: $0 " 1>&2 + exit 1 +fi + +XCODE_TOOLS_DMG=$(make_absolute_path "$1" $(get_exec_dir)) + +mkdir -p $BUILD_DIR +pushd $BUILD_DIR &>/dev/null + +require git +require $MAKE +require cpio + +[ -n "$CC" ] && require $CC +[ -n "$CXX" ] && require $CXX + +# build xar +build_xar + +# build pbzx +get_sources https://github.com/tpoechtrager/pbzx.git master + +if [ $f_res -eq 1 ]; then + pushd $CURRENT_BUILD_PROJECT_NAME &>/dev/null + mkdir -p $TARGET_DIR_SDK_TOOLS/bin + verbose_cmd $CC -I $TARGET_DIR/include -L $TARGET_DIR/lib pbzx.c \ + -o $TARGET_DIR_SDK_TOOLS/bin/pbzx -llzma -lxar \ + -Wl,-rpath,$TARGET_DIR/lib + build_success + popd &>/dev/null +fi + +# build 7z +get_sources https://github.com/tpoechtrager/p7zip.git master + +if [ $f_res -eq 1 ]; then + pushd $CURRENT_BUILD_PROJECT_NAME &>/dev/null + + if [ -n "$CC" ] && [ -n "$CXX" ]; then + [[ $CC == *clang* ]] && CC="$CC -Qunused-arguments" + [[ $CXX == *clang* ]] && CXX="$CXX -Qunused-arguments" + $MAKE 7z -j $JOBS CC="$CC" CXX="$CXX -std=gnu++98" + else + $MAKE 7z -j $JOBS CXX="c++ -std=gnu++98" + fi + + $MAKE install DEST_HOME=$TARGET_DIR_SDK_TOOLS + find $TARGET_DIR_SDK_TOOLS/share -type f -exec chmod 0664 {} \; + find $TARGET_DIR_SDK_TOOLS/share -type d -exec chmod 0775 {} \; + popd &>/dev/null + build_success +fi + +create_tmp_dir + +pushd $TMP_DIR &>/dev/null + +echo "" +echo "Unpacking $XCODE_TOOLS_DMG ..." +XCODE_TOOLS_PKG="Command Line Developer Tools/Command Line Tools.pkg" +LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TARGET_DIR/lib \ + verbose_cmd "$TARGET_DIR_SDK_TOOLS/bin/7z x \"$XCODE_TOOLS_DMG\" \"$XCODE_TOOLS_PKG\"" + +echo "" +echo "Unpacking $XCODE_TOOLS_PKG ..." +mkdir "$TMP_DIR/pkg_data" +LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TARGET_DIR/lib \ + verbose_cmd "$TARGET_DIR/bin/xar -xf \"$XCODE_TOOLS_PKG\" -C $TMP_DIR/pkg_data" + +echo "" +echo "Processing packages ..." +mkdir "$TMP_DIR/out" +for PKG in $TMP_DIR/pkg_data/*.pkg; do + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TARGET_DIR/lib \ + verbose_cmd "$TARGET_DIR/SDK/tools/bin/pbzx -n \"$PKG/Payload\" | cpio -i -D $TMP_DIR/out" +done + + +popd &>/dev/null # TMP_DIR +popd &>/dev/null # BUILD_DIR + +echo "" + +XCODE_TOOLS_DIR="$TMP_DIR/out/Library/Developer/CommandLineTools" \ + ./tools/gen_sdk_package_tools.sh