diff --git a/.github/actions/build-setup-common/action.yml b/.github/actions/build-setup-common/action.yml new file mode 100644 index 000000000000..4165df199f1e --- /dev/null +++ b/.github/actions/build-setup-common/action.yml @@ -0,0 +1,107 @@ +# Part of the Carbon Language project, under the Apache License v2.0 with LLVM +# Exceptions. See /LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: Setup build environment (macOS) +inputs: + matrix_runner: + required: true + remote_cache_upload: + required: true +runs: + using: composite + steps: + # Setup Python and related tools. + - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 + with: + # Match the min version listed in docs/project/contribution_tools.md + # or the oldest version available on the OS. + python-version: + ${{ inputs.matrix_runner == 'macos-14' && '3.11' || '3.9' }} + + - uses: ./.github/actions/build-setup-macos + if: startsWith(inputs.matrix_runner, 'macos') + with: + matrix_runner: ${{ inputs.matrix_runner }} + + - uses: ./.github/actions/build-setup-ubuntu + if: startsWith(inputs.matrix_runner, 'ubuntu') + + # Print the various tool paths and versions to help in debugging. + - name: Print tool debugging info + shell: bash + run: | + echo '*** PATH' + echo $PATH + echo '*** bazelisk' + which bazelisk + bazelisk --version + echo '*** run_bazel.py' + ./scripts/run_bazel.py --version + echo '*** python' + which python + python --version + echo '*** clang' + which clang + clang --version + echo '*** clang++' + which clang++ + clang++ --version + + # Add our bazel configuration and print basic info to ease debugging. + - name: Configure Bazel and print info + env: + # Add a cache version for changes that bazel won't otherwise detect, + # like llvm version changes. + CACHE_VERSION: 1 + shell: bash + run: | + cat >user.bazelrc <> $GITHUB_PATH + echo '*** ls "${LLVM_PATH}"' + ls "${LLVM_PATH}" + echo '*** ls "${LLVM_PATH}/bin"' + ls "${LLVM_PATH}/bin" diff --git a/.github/actions/build-setup-ubuntu/action.yml b/.github/actions/build-setup-ubuntu/action.yml new file mode 100644 index 000000000000..d09c2d5fbb6e --- /dev/null +++ b/.github/actions/build-setup-ubuntu/action.yml @@ -0,0 +1,66 @@ +# Part of the Carbon Language project, under the Apache License v2.0 with LLVM +# Exceptions. See /LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +name: Setup build environment (Ubuntu) +runs: + using: composite + steps: + # Ubuntu images start with 23GB available, and this adds 14GB more. For + # comparison, MacOS images have >100GB free. + # + # Although we could delete more, if we run into a limit, not deleting + # everything provides a little flexibility to get space while trying + # to shrink the build. + - name: Free up disk space + uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 + with: + android: true + dotnet: true + haskell: true + + # Cache and install a recent version of LLVM. This uses the GitHub action + # cache to avoid directly downloading on each iteration and improve + # reliability. + - name: Cache LLVM and Clang installation + id: cache-llvm-ubuntu + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + env: + cache-name: cache-llvm + with: + path: ~/llvm + key: LLVM-16-Cache-ubuntu-${{ runner.arch }} + + - name: Download LLVM and Clang installation + if: steps.cache-llvm-ubuntu.outputs.cache-hit != 'true' + shell: bash + run: | + cd ~ + LLVM_RELEASE=clang+llvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04 + echo "*** Downloading $LLVM_RELEASE" + wget --show-progress=off "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/$LLVM_RELEASE.tar.xz" + echo "*** Extracting $LLVM_RELEASE" + tar -xJf "$LLVM_RELEASE.tar.xz" + echo "*** Moving to 'llvm'" + mv "$LLVM_RELEASE" llvm + echo "*** Testing `clang++ --version`" + ~/llvm/bin/clang++ --version + + # The installation contains *huge* parts of LLVM we don't need for the + # toolchain. Prune them here to keep our cache small. + echo "*** Cleaning the 'llvm' directory" + rm llvm/lib/{*.a,*.so,*.so.*,*.bc} + rm llvm/bin/{flang-*,mlir-*,clang-{scan-deps,check,repl},*-test,llvm-{lto*,reduce,bolt*,exegesis,jitlink},bugpoint,opt,llc} + echo "*** Size of the 'llvm' directory" + du -hs llvm + + - name: Setup LLVM and Clang paths + shell: bash + run: | + LLVM_PATH=~/llvm + echo "Using ${LLVM_PATH}" + echo "${LLVM_PATH}/bin" >> $GITHUB_PATH + echo '*** ls "${LLVM_PATH}"' + ls "${LLVM_PATH}" + echo '*** ls "${LLVM_PATH}/bin"' + ls "${LLVM_PATH}/bin" diff --git a/.github/workflows/nightly-release.yaml b/.github/workflows/nightly-release.yaml new file mode 100644 index 000000000000..fef73a42a504 --- /dev/null +++ b/.github/workflows/nightly-release.yaml @@ -0,0 +1,101 @@ +# Part of the Carbon Language project, under the Apache License v2.0 with LLVM +# Exceptions. See /LICENSE for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +# +# This workflow creates a GitHub "release" of a nightly build of the project. +# +# Note: This is just an initial rough attempt, there is a lot of future work +# needed here. A brief summary of TODOs: +# +# - Configure a nice release notes template and switch to generating the title +# and notes instead of hard coding them. +# +# - Do some amount of testing prior to building and uploading the release. +# - Tempting to try to examine existing testing workflow, but maybe better to +# allow re-using any complex parts and do our own testing. That would, for +# example, allow us to narrow or expand the set of tests uses for +# pre-release testing to potentially be different from continuous testing. +# - Some questions around what to do in the event of a failure... error? Where +# does the error go? Create a draft, unpublished release instead? +# +# - Build artifacts for all the different OSes we have GitHub runners for rather +# than just x86 Linux. + +name: Nightly Release + +on: + schedule: + - cron: '0 2 * * *' + # Enable manual runs for testing or manually (re-)creating a nightly release. + workflow_dispatch: + # FIXME: + push: + branches: [action-test] + +permissions: + contents: write # For creating and uploading to releases. + +jobs: + release: + runs-on: ubuntu-22.04 + steps: + - name: Harden Runner + uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 + with: + egress-policy: audit + + - name: Checkout branch + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - name: Set up remote cache access + env: + REMOTE_CACHE_KEY: ${{ secrets.CARBON_BUILDS_GITHUB }} + run: | + echo "$REMOTE_CACHE_KEY" | base64 -d > $HOME/remote_cache_key.json + echo "remote_cache_upload=--google_credentials=$HOME/remote_cache_key.json" \ + >> $GITHUB_ENV + + - uses: ./.github/actions/build-setup-common + with: + matrix_runner: ubuntu-22.04 + remote_cache_upload: ${{ env.remote_cache_upload }} + + - name: Get nightly date + run: | + echo "nightly_date=$(date '+%Y.%m.%d')" >> $GITHUB_ENV + + - name: Build release + env: + # 'libtool_check_unique failed to generate' workaround. + # https://github.com/bazelbuild/bazel/issues/14113#issuecomment-999794586 + BAZEL_USE_CPP_ONLY_TOOLCHAIN: 1 + run: | + ./scripts/run_bazel.py \ + --attempts=5 --jobs-on-last-attempt=4 \ + build -c opt --remote_download_toplevel \ + --pre_release=nightly --nightly_date=${{ env.nightly_date }} \ + //toolchain/install:prefix_root/bin/carbon \ + //toolchain/install:carbon_toolchain_tar_rule + + - name: Extract the release version + run: | + # Make sure we can run the toolchain to get the version. + ./bazel-bin/toolchain/install/prefix_root/bin/carbon version + + # Now stash it in a variable and export it. + VERSION=$( \ + ./bazel-bin/toolchain/install/prefix_root/bin/carbon version \ + | cut -d' ' -f5 | cut -d'+' -f1) + echo "release_version=$VERSION" >> $GITHUB_ENV + + - name: Create the release + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release create \ + --title "Nightly build ${{ env.nightly_date }}" \ + --notes 'A nightly development build of Carbon.' \ + -d \ + --prerelease \ + v${{ env.release_version }} \ + "bazel-bin/toolchain/install/carbon_toolchain-${{ env.release_version }}.tar.bz2#Linux x86-64 binaries - Ubuntu-22.04 (tar.bz2)" diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 4b5f1845ea99..48080f053b1a 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -37,29 +37,12 @@ jobs: build_mode: clang-tidy runs-on: ${{ matrix.runner }} - env: - os: ${{ startsWith(matrix.runner, 'ubuntu') && 'ubuntu' || 'macos' }} - steps: - name: Harden Runner uses: step-security/harden-runner@63c24ba6bd7ba022e95695ff85de572c04a18142 # v2.7.0 with: egress-policy: audit - # Ubuntu images start with 23GB available, and this adds 14GB more. For - # comparison, MacOS images have >100GB free. - # - # Although we could delete more, if we run into a limit, not deleting - # everything provides a little flexibility to get space while trying - # to shrink the build. - - name: Free up disk space (Ubuntu) - if: env.os == 'ubuntu' - uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # v1.3.1 - with: - android: true - dotnet: true - haskell: true - # Checkout the pull request head or the branch. - name: Checkout pull request if: github.event_name == 'pull_request' @@ -81,149 +64,6 @@ jobs: has_code: - '!{**/*.md,LICENSE,CODEOWNERS,.git*}' - # Setup Python and related tools. - - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0 - if: steps.filter.outputs.has_code == 'true' - with: - # Match the min version listed in docs/project/contribution_tools.md - # or the oldest version available on the OS. - python-version: ${{ matrix.runner == 'macos-14' && '3.11' || '3.9' }} - - # Install and cache LLVM 16 from Homebrew. - # TODO: We can potentially remove this and simplify things when the - # Homebrew version of LLVM updates to 16 here: - # https://github.com/actions/runner-images/blob/main/images/macos/macos-12-Readme.md - - name: Cache Homebrew (macOS) - if: steps.filter.outputs.has_code == 'true' && env.os == 'macos' - id: cache-homebrew-macos - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 - env: - cache-name: cache-homebrew - with: - # Cover all the critical parts of Homebrew here. Homebrew on Arm macOS - # uses its own prefix making this easy to cover, but we need a few - # different paths for Intel. - path: | - ${{ - runner.arch == 'ARM64' && '/opt/homebrew' || - ' - /usr/local/Homebrew - /usr/local/Cellar - /usr/local/Frameworks - /usr/local/bin - /usr/local/opt - ' - }} - # Note the key needs to include all the packages we're adding. - key: Homebrew-Cache-${{ matrix.runner }}-${{ runner.arch }} - - - name: Install LLVM and Clang with Homebrew (macOS) - if: - steps.filter.outputs.has_code == 'true' && env.os == 'macos' && - steps.cache-homebrew-macos.outputs.cache-hit != 'true' - run: | - echo '*** Prune brew leaves' - # We prune all the leaf packages to have a minimal environment. This - # both minimizes the install space and avoids accidental dependencies - # on installed packages. - brew leaves - LEAVES=$(brew leaves | egrep -v '^(bazelisk|gh|git|git-lfs|gnu-tar|go@.*|jq|pipx|node@.*|openssl@.*|wget|yq|zlib)$') - brew uninstall -f --ignore-dependencies $LEAVES - echo '*** Installing LLVM deps' - brew install --force-bottle --only-dependencies llvm@16 - echo '*** Installing LLVM itself' - brew install --force-bottle --force --verbose llvm@16 - echo '*** brew info llvm@16' - brew info llvm@16 - echo '*** brew autoremove' - brew autoremove - echo '*** brew info' - brew info - echo '*** brew leaves' - brew leaves - echo '*** brew config' - brew config - - - name: Setup LLVM and Clang (macOS) - if: steps.filter.outputs.has_code == 'true' && env.os == 'macos' - run: | - LLVM_PATH="$(brew --prefix llvm@16)" - echo "Using ${LLVM_PATH}" - echo "${LLVM_PATH}/bin" >> $GITHUB_PATH - echo '*** ls "${LLVM_PATH}"' - ls "${LLVM_PATH}" - echo '*** ls "${LLVM_PATH}/bin"' - ls "${LLVM_PATH}/bin" - - # Cache and install a recent version of LLVM. This uses the GitHub action - # cache to avoid directly downloading on each iteration and improve - # reliability. - - name: Cache LLVM and Clang installation (Ubuntu) - if: steps.filter.outputs.has_code == 'true' && env.os == 'ubuntu' - id: cache-llvm-ubuntu - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 - env: - cache-name: cache-llvm - with: - path: ~/llvm - key: LLVM-16-Cache-${{ env.os }}-${{ runner.arch }} - - - name: Download LLVM and Clang installation (Ubuntu) - if: - steps.filter.outputs.has_code == 'true' && env.os == 'ubuntu' && - steps.cache-llvm-ubuntu.outputs.cache-hit != 'true' - run: | - cd ~ - LLVM_RELEASE=clang+llvm-16.0.4-x86_64-linux-gnu-ubuntu-22.04 - echo "*** Downloading $LLVM_RELEASE" - wget --show-progress=off "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/$LLVM_RELEASE.tar.xz" - echo "*** Extracting $LLVM_RELEASE" - tar -xJf "$LLVM_RELEASE.tar.xz" - echo "*** Moving to 'llvm'" - mv "$LLVM_RELEASE" llvm - echo "*** Testing `clang++ --version`" - ~/llvm/bin/clang++ --version - - # The installation contains *huge* parts of LLVM we don't need for the - # toolchain. Prune them here to keep our cache small. - echo "*** Cleaning the 'llvm' directory" - rm llvm/lib/{*.a,*.so,*.so.*,*.bc} - rm llvm/bin/{flang-*,mlir-*,clang-{scan-deps,check,repl},*-test,llvm-{lto*,reduce,bolt*,exegesis,jitlink},bugpoint,opt,llc} - echo "*** Size of the 'llvm' directory" - du -hs llvm - - - name: Setup LLVM and Clang paths (Ubuntu) - if: steps.filter.outputs.has_code == 'true' && env.os == 'ubuntu' - run: | - LLVM_PATH=~/llvm - echo "Using ${LLVM_PATH}" - echo "${LLVM_PATH}/bin" >> $GITHUB_PATH - echo '*** ls "${LLVM_PATH}"' - ls "${LLVM_PATH}" - echo '*** ls "${LLVM_PATH}/bin"' - ls "${LLVM_PATH}/bin" - - # Print the various tool paths and versions to help in debugging. - - name: Print tool debugging info - if: steps.filter.outputs.has_code == 'true' - run: | - echo '*** PATH' - echo $PATH - echo '*** bazelisk' - which bazelisk - bazelisk --version - echo '*** run_bazel.py' - ./scripts/run_bazel.py --version - echo '*** python' - which python - python --version - echo '*** clang' - which clang - clang --version - echo '*** clang++' - which clang++ - clang++ --version - # Disable uploads when the remote cache is read-only. - name: Set up remote cache access (read-only) if: @@ -245,63 +85,11 @@ jobs: echo "remote_cache_upload=--google_credentials=$HOME/remote_cache_key.json" \ >> $GITHUB_ENV - # Add our bazel configuration and print basic info to ease debugging. - - name: Configure Bazel and print info + - uses: ./.github/actions/build-setup-common if: steps.filter.outputs.has_code == 'true' - env: - # Add a cache version for changes that bazel won't otherwise detect, - # like llvm version changes. - CACHE_VERSION: 1 - run: | - cat >user.bazelrc <