From a6a51cad01a5e0a6400f39e254dd7f313e80a45d Mon Sep 17 00:00:00 2001 From: Tpt Date: Mon, 17 Jan 2022 22:22:05 +0100 Subject: [PATCH] Improves build and release scripts --- .github/FUNDING.yml | 2 +- .github/workflows/artifacts.yml | 103 ++++++++++++++++++++++++++- .github/workflows/manylinux_build.sh | 8 +++ .github/workflows/release.yml | 101 ++++++++++++-------------- .github/workflows/tests.yml | 67 ++--------------- 5 files changed, 163 insertions(+), 118 deletions(-) create mode 100644 .github/workflows/manylinux_build.sh diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml index 9a4dc9a6..7e9239cb 100644 --- a/.github/FUNDING.yml +++ b/.github/FUNDING.yml @@ -1 +1 @@ -github: [Tpt] +github: [ Tpt ] diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 4606d309..ba2fcc36 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -6,7 +6,7 @@ on: - master jobs: - python: + python_doc: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -38,6 +38,104 @@ jobs: git push working-directory: ./website + binary_linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - run: cargo build --release + working-directory: ./server + - uses: actions/upload-artifact@v2 + with: + name: oxigraph_server_x86_64_linux_gnu + path: target/release/oxigraph_server + + binary_mac: + runs-on: macos-latest + env: + DEVELOPER_DIR: '/Applications/Xcode.app/Contents/Developer' + SDKROOT: '/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk' + MACOSX_DEPLOYMENT_TARGET: '10.9' + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - run: rustup update + - run: cargo build --release + working-directory: ./server + - uses: actions/upload-artifact@v2 + with: + name: oxigraph_server_x86_64_apple + path: target/release/oxigraph_server + + binary_windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - run: rustup update + - run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse + - run: cargo build --release + working-directory: ./server + - uses: actions/upload-artifact@v2 + with: + name: oxigraph_server_x86_64_windows_msvc + path: target/release/oxigraph_server.exe + + wheel_linux: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - run: sed 's/%arch%/x86_64/g' .github/workflows/manylinux_build.sh > .github/workflows/manylinux_build_script.sh + - run: docker run -v "$(pwd)":/workdir --platform linux/x86_64 quay.io/pypa/manylinux2014_x86_64 /bin/bash /workdir/.github/workflows/manylinux_build_script.sh + - uses: actions/upload-artifact@v2 + with: + name: pyoxigraph_x86_64_linux + path: target/wheels/*.whl + + wheel_mac: + runs-on: macos-latest + env: + DEVELOPER_DIR: '/Applications/Xcode.app/Contents/Developer' + SDKROOT: '/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk' + MACOSX_DEPLOYMENT_TARGET: '10.9' + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/setup-python@v2 + with: + python-version: 3.7 + - run: rustup update && rustup target add aarch64-apple-darwin + - run: pip install maturin + - run: maturin build --release -m python/Cargo.toml --universal2 + - uses: actions/upload-artifact@v2 + with: + name: pyoxigraph_wheel_universal2_mac + path: target/wheels/*.whl + + wheel_windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + with: + submodules: true + - uses: actions/setup-python@v2 + with: + python-version: 3.7 + - run: rustup update + - run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse + - run: pip install maturin + - run: maturin build --release -m python/Cargo.toml + - uses: actions/upload-artifact@v2 + with: + name: pyoxigraph_wheel_x86_64_windows + path: target/wheels/*.whl + docker: runs-on: ubuntu-latest steps: @@ -45,8 +143,7 @@ jobs: with: submodules: 'recursive' - uses: docker/setup-buildx-action@v1 - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + - uses: docker/login-action@v1 with: registry: ghcr.io username: ${{github.actor}} diff --git a/.github/workflows/manylinux_build.sh b/.github/workflows/manylinux_build.sh new file mode 100644 index 00000000..00ad5423 --- /dev/null +++ b/.github/workflows/manylinux_build.sh @@ -0,0 +1,8 @@ +cd /workdir +yum -y install centos-release-scl-rh +yum -y install llvm-toolset-7.0 +source scl_source enable llvm-toolset-7.0 +curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal +curl --verbose -L "https://github.com/PyO3/maturin/releases/latest/download/maturin-%arch%-unknown-linux-musl.tar.gz" | tar -xz -C /usr/local/bin +export PATH="${PATH}:/root/.cargo/bin:/opt/python/cp37-cp37m/bin:/opt/python/cp38-cp38/bin:/opt/python/cp39-cp39/bin:/opt/python/cp310-cp310/bin" +maturin build --release --no-sdist -m python/Cargo.toml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c3453a5a..66af914f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -2,7 +2,7 @@ name: Release artifacts on: release: - types: [published] + types: [ published ] jobs: push_server_to_docker_registry: @@ -18,13 +18,11 @@ jobs: images: | ${{ github.repository }} ghcr.io/${{ github.repository }} - - name: Login to DockerHub - uses: docker/login-action@v1 + - uses: docker/login-action@v1 with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} - - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + - uses: docker/login-action@v1 with: registry: ghcr.io username: ${{github.actor}} @@ -40,7 +38,7 @@ jobs: cache-from: type=gha cache-to: type=gha,mode=max - publish_rocksdb_crate: + publish_crates: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 @@ -52,72 +50,64 @@ jobs: CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} - run: cargo publish working-directory: ./rocksdb-sys - - publish_lib_crate: - runs-on: ubuntu-latest - needs: publish_rocksdb_crate - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - run: rustup update - - run: cargo login $CRATES_IO_TOKEN - env: - CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} + - run: cargo publish + working-directory: ./lib/oxrdf + - run: cargo publish + working-directory: ./lib/sparesults + - run: cargo publish + working-directory: ./lib/spargebra - run: cargo publish working-directory: ./lib - - publish_server_crate: - runs-on: ubuntu-latest - needs: publish_lib_crate - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - run: rustup update - - run: cargo login $CRATES_IO_TOKEN - env: - CRATES_IO_TOKEN: ${{ secrets.CRATES_IO_TOKEN }} - run: cargo publish working-directory: ./server - python_wheel_linux: + publish_pypi_linux: runs-on: ubuntu-latest - needs: publish_lib_crate + needs: publish_lib_crates + strategy: + matrix: + architecture: [ "x86_64", "aarch64" ] steps: - uses: actions/checkout@v2 with: submodules: true - run: sed -i 's/path="..\/lib", //g' python/Cargo.toml - - uses: messense/maturin-action@v1 + - run: sed 's/%arch%/${{ matrix.architecture }}/g' .github/workflows/manylinux_build.sh > .github/workflows/manylinux_build_script.sh + - run: docker run -v "$(pwd)":/workdir --platform linux/${{ matrix.architecture }} quay.io/pypa/manylinux2014_${{ matrix.architecture }} /bin/bash /workdir/.github/workflows/manylinux_build_script.sh + - uses: pypa/gh-action-pypi-publish@release/v1 with: - manylinux: 2014 - container: messense/manylinux2014-cross:x86_64 - command: publish - args: -m python/Cargo.toml --cargo-extra-args="--no-default-features --features vendored" --no-sdist -u __token__ -p ${{ secrets.PYPI_PASSWORD }} + user: __token__ + password: ${{ secrets.PYPI_PASSWORD }} + packages_dir: target/wheels publish_pypi_mac: runs-on: macos-latest - needs: publish_lib_crate + needs: publish_lib_crates + env: + DEVELOPER_DIR: '/Applications/Xcode.app/Contents/Developer' + SDKROOT: '/Applications/Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk' + MACOSX_DEPLOYMENT_TARGET: '10.9' steps: - uses: actions/checkout@v2 with: submodules: true - - uses: messense/maturin-action@v1 + - uses: actions/setup-python@v2 with: - command: publish - args: -m python/Cargo.toml --universal2 -u __token__ -p ${{ secrets.PYPI_PASSWORD }} - - uses: messense/maturin-action@v1 + python-version: 3.7 + - run: rustup update && rustup target add aarch64-apple-darwin + - run: pip install maturin + - run: sed -i 's/path="..\/lib", //g' python/Cargo.toml + - run: maturin build --release -m python/Cargo.toml --universal2 + - run: maturin build --release --no-sdist -m python/Cargo.toml + - uses: pypa/gh-action-pypi-publish@release/v1 with: - command: publish - args: -m python/Cargo.toml --no-sdist -u __token__ -p ${{ secrets.PYPI_PASSWORD }} + user: __token__ + password: ${{ secrets.PYPI_PASSWORD }} + packages_dir: target/wheels - python_wheel_windows: + publish_pypi_windows: runs-on: windows-latest - needs: publish_lib_crate - strategy: - matrix: - architecture: ["x86", "x64"] + needs: publish_lib_crates steps: - uses: actions/checkout@v2 with: @@ -125,13 +115,16 @@ jobs: - uses: actions/setup-python@v2 with: python-version: 3.7 - architecture: ${{ matrix.architecture }} + - run: rustup update + - run: pip install maturin - run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse - - uses: messense/maturin-action@v1 + - run: sed -i 's/path="..\/lib", //g' python/Cargo.toml + - run: maturin build --release --no-sdist -m python/Cargo.toml + - uses: pypa/gh-action-pypi-publish@release/v1 with: - command: publish - target: ${{ matrix.architecture }} - args: -m python/Cargo.toml --no-sdist -u __token__ -p ${{ secrets.PYPI_PASSWORD }} + user: __token__ + password: ${{ secrets.PYPI_PASSWORD }} + packages_dir: target/wheels publish_npm: runs-on: ubuntu-latest diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d3abc966..0f54330e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -36,13 +36,13 @@ jobs: test_linux: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - with: - submodules: true - - run: rustup update - - run: cargo test --all-features - env: - RUST_BACKTRACE: 1 + - uses: actions/checkout@v2 + with: + submodules: true + - run: rustup update + - run: cargo test --all-features + env: + RUST_BACKTRACE: 1 address_sanitizer: runs-on: ubuntu-latest @@ -99,56 +99,3 @@ jobs: working-directory: ./python/docs - run: source ../venv/bin/activate && sphinx-build -M html . build working-directory: ./python/docs - - python_wheel_linux: - runs-on: ubuntu-latest - needs: python - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - uses: messense/maturin-action@v1 - with: - manylinux: 2014 - container: messense/manylinux2014-cross:x86_64 - command: build - args: -m python/Cargo.toml - - run: pip install pyoxigraph --no-index --find-links ../../target/wheels && python -m unittest - working-directory: ./python/tests - - python_wheel_mac: - runs-on: macos-latest - needs: python - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - uses: messense/maturin-action@v1 - with: - command: build - args: -m python/Cargo.toml --universal2 - - run: pip install pyoxigraph --no-index --find-links ../../target/wheels && python -m unittest - working-directory: ./python/tests - - python_wheel_windows: - runs-on: windows-latest - needs: python - strategy: - matrix: - architecture: ["x86", "x64"] - steps: - - uses: actions/checkout@v2 - with: - submodules: true - - uses: actions/setup-python@v2 - with: - python-version: 3.7 - architecture: ${{ matrix.architecture }} - - run: Remove-Item -LiteralPath "C:\msys64\" -Force -Recurse - - uses: messense/maturin-action@v1 - with: - command: build - target: ${{ matrix.architecture }} - args: -m python/Cargo.toml - - run: pip install pyoxigraph --no-index --find-links ../../target/wheels && python -m unittest - working-directory: ./python/tests