Merge pull request #254 from rokm/xcode_cmdline_tools

Add scripts for extracting SDKs from Command Line Tools for Xcode

README.md

@@ -159,7 +159,10 @@ use these variants unless you know what you're doing.
 **[Please ensure you have read and understood the Xcode license
    terms before continuing.](https://www.apple.com/legal/sla/docs/xcode.pdf)**
 
-##### Packaging the SDK on macOS: #####
+The SDKs can be extracted either from full Xcode or from Command Line
+Tools for Xcode.
+
+##### Packaging the SDK on macOS (Xcode): #####
 
 1. [Download Xcode: https://developer.apple.com/download/more] \*\*
 2. [Mount Xcode.dmg (Open With -> DiskImageMounter) \*\*\*]
@@ -212,6 +215,34 @@ An SSD is recommended for this method.
 6. Copy or move the SDK into the tarballs/ directory
 
 
+##### Packaging the SDK from Xcode Command Line Tools on macOS: #####
+
+1. [Download Xcode Command Line Tools: https://developer.apple.com/download/more] \*\*\*\*
+2. [Mount Command_Line_Tools_for_Xcode.dmg (Open With -> DiskImageMounter)]
+3. [Install "Command Line Tools.pkg" (Open With -> Installer)]
+3. Run: `./tools/gen_sdk_package_tools.sh` (from the OSXCross package)
+4. Copy the packaged SDK (\*.tar.\* or \*.pkg) on a USB Stick
+5. (On Linux/BSD) Copy or move the SDK into the tarballs/ directory of
+   OSXCross.
+
+\*\*\*\*
+-- Xcode command line tools 12.x are known to work.
+
+Steps 1. to 3. can be skipped if you have Xcode Command line tools
+already installed (e.g., auto-installed by running `git` or `gcc`
+command from command-line).
+
+##### Packing the SDK from from Xcode Command Line Tools on Linux: #####
+
+This method may require up to 25 GB of free disk space.
+An SSD is recommended for this method.
+
+1. Download Xcode Command Line Tools like described in 'Packaging the SDK from Xcode Command Line Tools on macOS'
+2. Install `clang`, `make`, `libssl-devel`, `lzma-devel` and `libxml2-devel`
+3. Run `./tools/gen_sdk_package_tools_dmg.sh <command_line_tools_for_xcode>.dmg`
+4. Copy or move the SDK into the tarballs/ directory
+
+
 ### USAGE EXAMPLES: ###
 
 ##### Example.  To compile a file called test.cpp, you can run: #####

tools/gen_sdk_package_tools.sh

@@ -0,0 +1,164 @@
+#!/usr/bin/env bash
+#
+# Package the macOS SDKs into a tar file to be used by `build.sh`.
+# As opposed to gen_sdk_package.sh, which is used for extraction of SDKs
+# from full Xcode version, gen_sdk_tools.sh extracts SDKs from Xcode
+# Command Line Tools.
+#
+# Tested with XCode Command Line Tools 12.x
+#
+
+export LC_ALL=C
+
+
+command -v gnutar &>/dev/null
+
+if [ $? -eq 0 ]; then
+  TAR=gnutar
+else
+  TAR=tar
+fi
+
+
+if [ -z "$SDK_COMPRESSOR" ]; then
+  command -v xz &>/dev/null
+
+  if [ $? -eq 0 ]; then
+    SDK_COMPRESSOR=xz
+    SDK_EXT="tar.xz"
+  else
+    SDK_COMPRESSOR=bzip2
+    SDK_EXT="tar.bz2"
+  fi
+fi
+
+case $SDK_COMPRESSOR in
+  "gz")
+    SDK_COMPRESSOR=gzip
+    SDK_EXT=".tar.gz"
+    ;;
+  "bzip2")
+    SDK_EXT=".tar.bz2"
+    ;;
+  "xz")
+    SDK_EXT=".tar.xz"
+    ;;
+  "zip")
+    SDK_EXT=".zip"
+    ;;
+  *)
+    echo "error: unknown compressor \"$SDK_COMPRESSOR\"" >&2
+    exit 1
+esac
+
+function compress()
+{
+  case $SDK_COMPRESSOR in
+    "zip")
+      $SDK_COMPRESSOR -q -5 -r - $1 > $2 ;;
+    *)
+      tar cf - $1 | $SDK_COMPRESSOR -5 - > $2 ;;
+  esac
+}
+
+
+function rreadlink()
+{
+  if [ ! -h "$1" ]; then
+    echo "$1"
+  else
+    local link="$(expr "$(command ls -ld -- "$1")" : '.*-> \(.*\)$')"
+    cd $(dirname $1)
+    rreadlink "$link" | sed "s|^\([^/].*\)\$|$(dirname $1)/\1|"
+  fi
+}
+
+
+if [ $(uname -s) != "Darwin" ]; then
+  if [ -z "$XCODE_TOOLS_DIR" ]; then
+    echo "This script must be run on macOS" 1>&2
+    echo "... Or with XCODE_TOOLS_DIR=... on Linux" 1>&2
+    exit 1
+  else
+    case "$XCODE_TOOLS_DIR" in
+      /*) ;;
+      *) XCODE_TOOLS_DIR="$PWD/$XCODE_TOOLS_DIR" ;;
+    esac
+  fi
+else
+  XCODE_TOOLS_DIR="/Library/Developer/CommandLineTools"
+fi
+
+if [ ! -d "$XCODE_TOOLS_DIR" ]; then
+  echo "cannot find Xcode Command Line Tools (XCODE_TOOLS_DIR=$XCODE_TOOLS_DIR)" 1>&2
+  exit 1
+fi
+
+echo -e "found Xcode Command Line Tools: $XCODE_TOOLS_DIR"
+
+WDIR=$(pwd)
+
+set -e
+
+pushd "$XCODE_TOOLS_DIR" &>/dev/null
+
+if [ -d "SDKs" ]; then
+  pushd "SDKs" &>/dev/null
+else
+  echo "$XCODE_TOOLS_DIR/SDKs does not exist"  1>&2
+  exit 1
+fi
+
+SDKS=$(ls | grep -E "^MacOSX11.*|^MacOSX10.*" | grep -v "Patch")
+
+if [ -z "$SDKS" ]; then
+    echo "No SDK found" 1>&2
+    exit 1
+fi
+
+# libc++ headers for C++11/C++14
+LIBCXXDIR="usr/include/c++/v1"
+
+# Manual directory
+MANDIR="usr/share/man"
+
+for SDK in $SDKS; do
+  echo -n "packaging $(echo "$SDK" | sed -E "s/(.sdk|.pkg)//g") SDK "
+  echo "(this may take several minutes) ..."
+
+  if [[ $SDK == *.pkg ]]; then
+    cp $SDK $WDIR
+    continue
+  fi
+
+  TMP=$(mktemp -d /tmp/XXXXXXXXXXX)
+  cp -r $(rreadlink $SDK) $TMP/$SDK &>/dev/null || true
+
+  pushd "$XCODE_TOOLS_DIR" &>/dev/null
+
+  mkdir -p $TMP/$SDK/usr/include/c++
+
+  # libc++ headers for C++11/C++14
+  if [ -d $LIBCXXDIR ]; then
+    cp -rf $LIBCXXDIR "$TMP/$SDK/usr/include/c++"
+  fi
+
+  if [ -d $MANDIR ]; then
+    mkdir -p $TMP/$SDK/usr/share/man
+    cp -rf $MANDIR/* $TMP/$SDK/usr/share/man
+  fi
+
+  popd &>/dev/null
+
+  pushd $TMP &>/dev/null
+  compress "*" "$WDIR/$SDK$SDK_EXT"
+  popd &>/dev/null
+
+  rm -rf $TMP
+done
+
+popd &>/dev/null
+popd &>/dev/null
+
+echo ""
+ls -lh | grep MacOSX

tools/gen_sdk_package_tools_dmg.sh

@@ -0,0 +1,104 @@
+#!/usr/bin/env bash
+#
+# Extract Xcode Command Line Tools .dmg, process "Command Line Tools.pkg"
+# and its sub-archives/packages, and run gen_sdk_package_tools.sh.
+#
+# Tested with XCode Command Line Tools 12.x
+#
+
+pushd "${0%/*}/.." &>/dev/null
+source tools/tools.sh
+
+require cpio
+
+if [ $PLATFORM == "Darwin" ]; then
+  echo "Use gen_sdk_package_tools.sh on macOS" 1>&2
+  exit 1
+fi
+
+if [ $# -eq 0 ]; then
+  echo "Usage: $0 <Command_Line_Tools_for_Xcode.dmg>" 1>&2
+  exit 1
+fi
+
+XCODE_TOOLS_DMG=$(make_absolute_path "$1" $(get_exec_dir))
+
+mkdir -p $BUILD_DIR
+pushd $BUILD_DIR &>/dev/null
+
+require git
+require $MAKE
+require cpio
+
+[ -n "$CC" ] && require $CC
+[ -n "$CXX" ] && require $CXX
+
+# build xar
+build_xar
+
+# build pbzx
+get_sources https://github.com/tpoechtrager/pbzx.git master
+
+if [ $f_res -eq 1 ]; then
+  pushd $CURRENT_BUILD_PROJECT_NAME &>/dev/null
+  mkdir -p $TARGET_DIR_SDK_TOOLS/bin
+  verbose_cmd $CC -I $TARGET_DIR/include -L $TARGET_DIR/lib pbzx.c \
+              -o $TARGET_DIR_SDK_TOOLS/bin/pbzx -llzma -lxar \
+              -Wl,-rpath,$TARGET_DIR/lib
+  build_success
+  popd &>/dev/null
+fi
+
+# build 7z
+get_sources https://github.com/tpoechtrager/p7zip.git master
+
+if [ $f_res -eq 1 ]; then
+  pushd $CURRENT_BUILD_PROJECT_NAME &>/dev/null
+
+  if [ -n "$CC" ] && [ -n "$CXX" ]; then
+    [[ $CC == *clang* ]] && CC="$CC -Qunused-arguments"
+    [[ $CXX == *clang* ]] && CXX="$CXX -Qunused-arguments"
+    $MAKE 7z -j $JOBS CC="$CC" CXX="$CXX -std=gnu++98"
+  else
+    $MAKE 7z -j $JOBS CXX="c++ -std=gnu++98"
+  fi
+
+  $MAKE install DEST_HOME=$TARGET_DIR_SDK_TOOLS
+  find $TARGET_DIR_SDK_TOOLS/share -type f -exec chmod 0664 {} \;
+  find $TARGET_DIR_SDK_TOOLS/share -type d -exec chmod 0775 {} \;
+  popd &>/dev/null
+  build_success
+fi
+
+create_tmp_dir
+
+pushd $TMP_DIR &>/dev/null
+
+echo ""
+echo "Unpacking $XCODE_TOOLS_DMG ..."
+XCODE_TOOLS_PKG="Command Line Developer Tools/Command Line Tools.pkg"
+LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TARGET_DIR/lib \
+  verbose_cmd "$TARGET_DIR_SDK_TOOLS/bin/7z x \"$XCODE_TOOLS_DMG\"  \"$XCODE_TOOLS_PKG\""
+
+echo ""
+echo "Unpacking $XCODE_TOOLS_PKG ..."
+mkdir "$TMP_DIR/pkg_data"
+LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TARGET_DIR/lib \
+  verbose_cmd "$TARGET_DIR/bin/xar -xf \"$XCODE_TOOLS_PKG\" -C $TMP_DIR/pkg_data"
+
+echo ""
+echo "Processing packages ..."
+mkdir "$TMP_DIR/out"
+for PKG in $TMP_DIR/pkg_data/*.pkg; do
+  LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$TARGET_DIR/lib \
+    verbose_cmd "$TARGET_DIR/SDK/tools/bin/pbzx -n \"$PKG/Payload\" | cpio -i -D $TMP_DIR/out"
+done
+
+
+popd &>/dev/null # TMP_DIR
+popd &>/dev/null # BUILD_DIR
+
+echo ""
+
+XCODE_TOOLS_DIR="$TMP_DIR/out/Library/Developer/CommandLineTools" \
+  ./tools/gen_sdk_package_tools.sh