From 0875afb3e1f756ad31ba66bc1a163b92fc6741ed Mon Sep 17 00:00:00 2001 From: Henner Zeller Date: Tue, 19 Dec 2023 05:24:27 -0800 Subject: [PATCH] Building on old platforms gets hard. Just relase the static builds. Building on a bunch of different platforms often creates trouble, and with a recent required switch to bazel5 (because of a newer protobuf version), the java requirements are not met anymore. However, we already build static binaries that are available for Linux x86 and arm64 (which are only slightly larger than the dynamically linked ones) so let's simplify the release process and just provide the static binaries. This makes it simple for everyone who wants to download and run Verible; for everyone else, they still can compile it themselve or use the version from their Linux distribution. Fixes #1871 --- .github/bin/generate-matrix.py | 43 ------ .github/bin/github-releases-setup.sh | 72 --------- .github/workflows/verible-ci.yml | 76 +--------- releasing/README.md | 21 --- releasing/bazel.dockerstage | 6 - releasing/build.sh | 21 --- releasing/centos.dockerfile | 10 -- releasing/centos/common/compiler.dockerstage | 11 -- releasing/centos/common/help2man.dockerstage | 2 - releasing/docker-run.sh | 150 ------------------- releasing/supported_bases.txt | 5 - releasing/ubuntu.dockerfile | 17 --- releasing/ubuntu/bionic/compiler.dockerstage | 12 -- releasing/ubuntu/common/compiler.dockerstage | 9 -- releasing/ubuntu/common/help2man.dockerstage | 2 - releasing/ubuntu/xenial/compiler.dockerstage | 12 -- 16 files changed, 1 insertion(+), 468 deletions(-) delete mode 100755 .github/bin/generate-matrix.py delete mode 100755 .github/bin/github-releases-setup.sh delete mode 100644 releasing/README.md delete mode 100644 releasing/bazel.dockerstage delete mode 100755 releasing/build.sh delete mode 100644 releasing/centos.dockerfile delete mode 100644 releasing/centos/common/compiler.dockerstage delete mode 100644 releasing/centos/common/help2man.dockerstage delete mode 100755 releasing/docker-run.sh delete mode 100644 releasing/supported_bases.txt delete mode 100644 releasing/ubuntu.dockerfile delete mode 100644 releasing/ubuntu/bionic/compiler.dockerstage delete mode 100644 releasing/ubuntu/common/compiler.dockerstage delete mode 100644 releasing/ubuntu/common/help2man.dockerstage delete mode 100644 releasing/ubuntu/xenial/compiler.dockerstage diff --git a/.github/bin/generate-matrix.py b/.github/bin/generate-matrix.py deleted file mode 100755 index fb887f3a8..000000000 --- a/.github/bin/generate-matrix.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 - -# Copyright 2021 The Verible Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -from pathlib import Path - -ARCHS = ["x86_64", "arm64"] -matrix = [] - -with (Path(__file__).parent.resolve().parent.parent / 'releasing' / 'supported_bases.txt').open('r') as fptr: - for items in [line.strip().split(':') for line in fptr.readlines()]: - for arch in ARCHS: - # Temporary workaround until #1926 gets properly resolved - if arch == ARCHS[1] and items[0] == "centos": - continue - matrix.append({ - 'os': items[0], - 'ver': items[1], - 'arch': arch, - 'link': 'dynamic', - }) - -for arch in ARCHS: - matrix.append({ - 'os': 'ubuntu', - 'ver': 'jammy', - 'arch': arch, - 'link': 'static', - }) - -print('::set-output name=matrix::' + str(matrix)) diff --git a/.github/bin/github-releases-setup.sh b/.github/bin/github-releases-setup.sh deleted file mode 100755 index ce5dbb7e9..000000000 --- a/.github/bin/github-releases-setup.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2020 The Verible Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# GitHub Releases -# --------------- -# Generate the GitHub Releases to deploy - -set -ex - -SCRIPT_DIR=$(dirname $0) -RELEASE_DIR=${1:-/tmp/releases} -rm -rf "$RELEASE_DIR/*" - -GIT_VERSION=${GIT_VERSION:-$(git rev-parse --short "$GITHUB_SHA")} - -PREFIX=$RELEASE_DIR/verible-$GIT_VERSION -PREFIX_BIN=$PREFIX/bin -PREFIX_DOC=$PREFIX/share/verible -PREFIX_MAN=$PREFIX/share/man/man1 -mkdir -p $PREFIX -mkdir -p $PREFIX_BIN -mkdir -p $PREFIX_DOC -mkdir -p $PREFIX_MAN - -# Binaries -bazel run :install ${BAZEL_OPTS} -c opt -- $PREFIX_BIN -for BIN in $PREFIX_BIN/*; do - ls -l $BIN - file $BIN - ldd $BIN || true # ignore the potential 'not a dynamic executable' errors for non-binaries -done - -${SCRIPT_DIR}/github-pages-setup.sh $PREFIX_BIN - -# Documentation -cp -a /tmp/pages/* $PREFIX_DOC -# Man pages -for app in syntax lint format diff obfuscate ; do - BIN_NAME="verible-verilog-${app}" - MAN_FILE="${PREFIX_MAN}/${BIN_NAME}.1" - help2man --help-option="--helpfull" --output="$MAN_FILE" "$PREFIX_BIN/$BIN_NAME" - gzip "$MAN_FILE" - - # Set up manpage for legacy tool-names as symbolic link to the real deal. - ln -s "${BIN_NAME}.1.gz" "${PREFIX_MAN}/verilog_${app}.1.gz" -done - -DISTRO_ARCH=$(uname -m) -DISTRO=$(lsb_release --short --id) -DISTRO_RELEASE=$(lsb_release --short --release) -DISTRO_CODENAME=$(lsb_release --short --codename | sed -e's/[^A-Za-z0-9]//g') -if [ "${TARGET_LINK}" = "static" ]; then - TARBALL=$RELEASE_DIR/verible-$GIT_VERSION-linux-static-$DISTRO_ARCH.tar.gz -else - TARBALL=$RELEASE_DIR/verible-$GIT_VERSION-$DISTRO-$DISTRO_RELEASE-$DISTRO_CODENAME-$DISTRO_ARCH.tar.gz -fi -( - cd $RELEASE_DIR - tar -zcvf $TARBALL verible-$GIT_VERSION -) diff --git a/.github/workflows/verible-ci.yml b/.github/workflows/verible-ci.yml index 469f0bfff..46e91233b 100644 --- a/.github/workflows/verible-ci.yml +++ b/.github/workflows/verible-ci.yml @@ -303,80 +303,6 @@ jobs: with: path: kythe_output/*.kzip - - Matrix: - runs-on: ubuntu-latest - name: Generate Build matrix - outputs: - matrix: ${{ steps.generate.outputs.matrix }} - - steps: - - - uses: actions/checkout@v3 - - - id: generate - run: ./.github/bin/generate-matrix.py - - - Build: - needs: Matrix - container: ubuntu:jammy - runs-on: [self-hosted, Linux, X64, gcp-custom-runners] - # Github actions resources are limited; don't build artifacts for all - # platforms on every pull request, only on push. - if: ${{github.event_name == 'push'}} - strategy: - fail-fast: false - matrix: - include: ${{ fromJson(needs.Matrix.outputs.matrix) }} - env: - MATRIX_OS: '${{ matrix.os }}:${{ matrix.ver }}' - LINK_TYPE: '${{ matrix.link }}' - ARCH: '${{ matrix.arch }}' - DOCKER_DATA_ROOT: "/root/.docker" - GHA_MACHINE_TYPE: "${{ matrix.arch == 'arm64' && 't2a-standard-8' || 'n2-highcpu-8' }}" - name: "Build ยท${{ matrix.arch }}:${{ matrix.os }}:${{ matrix.ver }}${{ matrix.link == 'static' && ':static' || '' }}" - - steps: - - - uses: actions/checkout@v3 - with: - # Download complete repository + tags - fetch-depth: 0 - - - name: Install and setup Docker - run: "apt -qqy update && apt -qqy --no-install-recommends install docker.io cgroupfs-mount crun fuse-overlayfs pigz ca-certificates git && cgroupfs-mount" - - - name: Main script - run: | - set -x - - mkdir -p "$DOCKER_DATA_ROOT" - dockerd \ - -s fuse-overlayfs \ - --add-runtime=crun=/usr/bin/crun \ - --default-runtime=crun \ - --config-file="" \ - --data-root=$DOCKER_DATA_ROOT > /dev/null 2>&1 & - while ! test -S /var/run/docker.sock; do echo "Waiting for Docker..." && sleep 1; done; docker info - trap "kill $(cat /var/run/docker.pid)" EXIT - - docker pull $MATRIX_OS - source ./.github/settings.sh - ARCH="$ARCH" LINK_TYPE="$LINK_TYPE" ./releasing/docker-run.sh $MATRIX_OS - - - name: ๐Ÿ“ค Upload artifact - uses: actions/upload-artifact@v2 - with: - path: releasing/out/verible-*.tar.gz - - - name: ๐Ÿ“ค Upload performance graphs - uses: actions/upload-artifact@v2 - if: success() || failure() - with: - name: "diag" - path: "**/plot_*.svg" - MacOsBuildDevTools: runs-on: macos-latest steps: @@ -546,7 +472,7 @@ jobs: Release: - needs: [ Check, Build, MacOsBuild, WindowsBuild, PrepareVSPlugin ] + needs: [ Check, MacOsBuild, WindowsBuild, PrepareVSPlugin ] runs-on: ubuntu-20.04 name: ๐Ÿ“ฆ Release if: ${{github.event_name == 'push' && (github.ref == 'refs/heads/main' || github.ref == 'refs/heads/master')}} diff --git a/releasing/README.md b/releasing/README.md deleted file mode 100644 index 5b81d2fc9..000000000 --- a/releasing/README.md +++ /dev/null @@ -1,21 +0,0 @@ -This directory is used to create Verible releases. - -The `docker-generate.sh` script creates a bunch of docker containers for -different Linux distributions which build the Verible tool. - -The `docker-run.sh` script actually runs the build inside the docker container -and extracts the built content. - -The general aim is to support all current versions of Ubuntu and CentOS (for -RedHat Enterprise Linux support). - -Currently supported Linux distributions are: - -- CentOS 7 -- Ubuntu Xenial (16.04 LTS) -- Ubuntu Bionic (18.04 LTS) -- Ubuntu Eoan (19.10) -- Ubuntu Focal (20.04 LTS) -- Ubuntu Groovy (20.10) - -Note that CentOS 8 was EOL on December 2021 but CentOS 7 is still supported until 2024. diff --git a/releasing/bazel.dockerstage b/releasing/bazel.dockerstage deleted file mode 100644 index 163eb9b49..000000000 --- a/releasing/bazel.dockerstage +++ /dev/null @@ -1,6 +0,0 @@ -# Install bazel -ARG BAZEL_VERSION -ARG ARCH - -RUN wget --no-verbose "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-linux-"${ARCH} -O /usr/bin/bazel; \ - chmod +x /usr/bin/bazel diff --git a/releasing/build.sh b/releasing/build.sh deleted file mode 100755 index 008fd97d7..000000000 --- a/releasing/build.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2021 The Verible Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -cd $(dirname "$0")/.. - -./.github/bin/github-releases-setup.sh /out diff --git a/releasing/centos.dockerfile b/releasing/centos.dockerfile deleted file mode 100644 index 92818674b..000000000 --- a/releasing/centos.dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -FROM centos:VERSION - -# Install basic tools -RUN yum install -y \ - file \ - git \ - redhat-lsb \ - tar \ - wget \ - python3 diff --git a/releasing/centos/common/compiler.dockerstage b/releasing/centos/common/compiler.dockerstage deleted file mode 100644 index 4493a6aa6..000000000 --- a/releasing/centos/common/compiler.dockerstage +++ /dev/null @@ -1,11 +0,0 @@ -# Link libstdc++ statically so people don't have to install devtoolset-8 -# just to use verible. -ENV BAZEL_LINKOPTS "-static-libstdc++:-lm -static-libstdc++:-lrt" -ENV BAZEL_LINKLIBS "-l%:libstdc++.a" - -# Get a newer GCC version -RUN yum install -y --nogpgcheck centos-release-scl -RUN yum install -y --nogpgcheck devtoolset-9 -RUN yum install -y --nogpgcheck glibc-static - -SHELL [ "scl", "enable", "devtoolset-9" ] diff --git a/releasing/centos/common/help2man.dockerstage b/releasing/centos/common/help2man.dockerstage deleted file mode 100644 index e3928affa..000000000 --- a/releasing/centos/common/help2man.dockerstage +++ /dev/null @@ -1,2 +0,0 @@ -# Install help2man -RUN yum install -y help2man diff --git a/releasing/docker-run.sh b/releasing/docker-run.sh deleted file mode 100755 index bd914c863..000000000 --- a/releasing/docker-run.sh +++ /dev/null @@ -1,150 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2020-2021 The Verible Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -set -e - -cd $(dirname "$0") - -# Get target OS and version - -if [ -z "${1}" ]; then - echo "Make sure that \$1 ($1) is set." - exit 1 -fi - -export TARGET=`echo "$1" | sed 's#:#-#g'` - -TARGET_VERSION=`echo $TARGET | cut -d- -f2` -TARGET_OS=`echo $TARGET | cut -d- -f1` -export TARGET_LINK=${LINK_TYPE:-"dynamic"} - -export TAG=${TAG:-$(git describe --match=v*)} - -if [ -z "${BAZEL_VERSION}" ]; then - echo "Make sure that \$BAZEL_VERSION ($BAZEL_VERSION) is set." - echo " (try 'source ../.github/settings.sh')" - exit 1 -fi -if [ -z "${BAZEL_OPTS}" ]; then - echo "Make sure that \$BAZEL_OPTS ($BAZEL_OPTS) is set." - echo " (try 'source ../.github/settings.sh')" - exit 1 -fi -if [ -z "${BAZEL_CXXOPTS}" ]; then - echo "Make sure that \$BAZEL_CXXOPTS ($BAZEL_CXXOPTS) is set." - echo " (try 'source ../.github/settings.sh')" - exit 1 -fi - -# Generate the Dockerfile - -OUT_DIR=${TARGET_OS}-${TARGET_VERSION} - -# Basic tools -mkdir -p "${OUT_DIR}" -sed "s#${TARGET_OS}:VERSION#${TARGET_OS}:${TARGET_VERSION}#g" ${TARGET_OS}.dockerfile > ${OUT_DIR}/Dockerfile - -case "$TARGET_OS" in - ubuntu) - # Compiler - [ "$TARGET_VERSION" = xenial ] || [ "$TARGET_VERSION" = bionic ] && _version="$TARGET_VERSION" || _version="common" - cat ${TARGET_OS}/${_version}/compiler.dockerstage >> ${OUT_DIR}/Dockerfile - - # Use local flex/bison on Jammy - if [ "$TARGET_VERSION" = jammy ]; then - BAZEL_OPTS="${BAZEL_OPTS} --//bazel:use_local_flex_bison" - echo 'RUN apt-get install -y flex bison' >> ${OUT_DIR}/Dockerfile - fi - ;; - centos) - # Compiler - cat ${TARGET_OS}/common/compiler.dockerstage >> ${OUT_DIR}/Dockerfile - ;; -esac - -if [ "${TARGET_LINK}" = "static" ]; then - BAZEL_OPTS="${BAZEL_OPTS} --config=create_static_linked_executables" - # Bazel link options with static libs cause error with - # --config=create_static_linked_executables and they are redundant - sed -i '/ENV BAZEL_LINK/d' ${OUT_DIR}/Dockerfile -fi - -# Bazel -cat bazel.dockerstage >> ${OUT_DIR}/Dockerfile -# help2man -cat ${TARGET_OS}/common/help2man.dockerstage >> ${OUT_DIR}/Dockerfile - -# ================================================================== - -REPO_SLUG=${GITHUB_REPOSITORY_SLUG:-google/verible} -GIT_DATE=${GIT_DATE:-$(git show -s --format=%ci)} -GIT_VERSION=${GIT_VERSION:-$(git describe --match=v*)} -GIT_HASH=${GIT_HASH:-$(git rev-parse HEAD)} - -# Build Verible - -cat >> ${OUT_DIR}/Dockerfile <