linux.yml

name: Linux (Ubuntu 22.04, Python 3.11)
on:
  workflow_dispatch:
  pull_request:
  merge_group:
  push:
    branches:
      - master
      - 'releases/**'

concurrency:
  # github.ref is not unique in post-commit
  group: ${{ github.event_name == 'push' && github.run_id || github.ref }}-linux
  cancel-in-progress: true

env:
  PYTHON_VERSION: '3.11'
  OV_BRANCH: ${{ github.base_ref || github.event.merge_group.base_ref || github.ref }}
  CMAKE_CXX_COMPILER_LAUNCHER: sccache
  CMAKE_C_COMPILER_LAUNCHER: sccache
  SCCACHE_IGNORE_SERVER_IO_ERROR: 1
  SCCACHE_SERVER_PORT: 35555
  SCCACHE_CACHE_SIZE: 30G
  SCCACHE_AZURE_KEY_PREFIX: tokenizers/ubuntu/22_04/x64
  ARTIFACTS_SHARE: '/mount/build-artifacts'
  BASE_PRODUCT_TYPE: public_linux_ubuntu_22_04_x86_64

permissions: read-all

jobs:
  openvino_download:
    name: Download prebuilt OpenVINO
    outputs:
      status: ${{ steps.openvino_download.outcome }}
      ov_wheel_source: ${{ steps.openvino_download.outputs.ov_wheel_source }}
      ov_version: ${{ steps.openvino_download.outputs.ov_version }}
      docker_tag: ${{ steps.get_docker_tag.outputs.docker_tag }}
    timeout-minutes: 10
    defaults:
      run:
        shell: bash
    runs-on: aks-linux-medium
    container:
      image: 'openvinogithubactions.azurecr.io/openvino_provider:0.1.0'
      volumes:
        - /mount:/mount
        - ${{ github.workspace }}:${{ github.workspace }}

    steps:
    - uses: openvinotoolkit/openvino/.github/actions/openvino_provider@master
      id: openvino_download
      with:
        platform: 'ubuntu22'
        commit_packages_to_provide: 'wheels,developer_package.tar.gz'
        revision: latest_available_commit

    - name: Clone docker tag from OpenVINO repo
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      with:
        repository: 'openvinotoolkit/openvino'
        path: 'openvino'
        ref: ${{ env.OV_BRANCH }}
        sparse-checkout: |
          .github/dockerfiles/docker_tag

    - name: Save docker tag to output
      id: get_docker_tag
      run: |
        docker_tag=$(cat openvino/.github/dockerfiles/docker_tag)
        echo "docker_tag=$docker_tag" >> $GITHUB_OUTPUT


  openvino_tokenizers_cpack:
    name: OpenVINO tokenizers cpack, BUILD_TYPE=${{ matrix.build_type }})
    strategy:
      matrix:
        build_type: [Release] # TODO: Add Debug build when OV provider is ready or use OV package
    needs: [ openvino_download ]
    if: |
      always() &&
      (needs.openvino_download.outputs.status == 'success')
    timeout-minutes: 45
    defaults:
      run:
        shell: bash
    runs-on: aks-linux-8-cores-16gb
    container:
      image: openvinogithubactions.azurecr.io/ov_build/ubuntu_22_04_x64:${{ needs.openvino_download.outputs.docker_tag }}
      volumes:
        - /mount:/mount
        - ${{ github.workspace }}:${{ github.workspace }}
      options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING
    env:
      CMAKE_GENERATOR: 'Ninja'
      OPENVINO_REPO: ${{ github.workspace }}/openvino
      INSTALL_DIR: ${{ github.workspace }}/openvino/install
      OV_TOKENIZERS_INSTALL_DIR: ${{ github.workspace }}/openvino_tokenizers/install
      OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}/openvino_tokenizers
      BUILD_DIR: ${{ github.workspace }}/openvino_tokenizers/build
      MANIFEST_PATH: ${{ github.workspace }}/manifest.yml

    steps:
      - name: Clone Openvino tokenizers
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          path: ${{ env.OPENVINO_TOKENIZERS_REPO }}
          
      - name: Generate product manifest
        id: create_manifest
        uses: openvinotoolkit/openvino/.github/actions/create_manifest@master
        with:
          repos: ${{ env.OPENVINO_TOKENIZERS_REPO }}
          product_type: ${{ env.BASE_PRODUCT_TYPE }}_${{ matrix.build_type }}
          target_arch: 'x86_64'
          build_type: ${{ matrix.build_type }}
          save_to: ${{ env.MANIFEST_PATH }}

      - name: Download OpenVINO package
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: ${{ needs.openvino_download.outputs.ov_artifact_name }}
          path: ${{ env.INSTALL_DIR }}
          merge-multiple: true

      #
      # Build
      #

      - name: CMake configure - tokenizers
        run: |
          source ${INSTALL_DIR}/setupvars.sh
          cmake -DOpenVINODeveloperPackage_DIR=${{ env.INSTALL_DIR }}/developer_package/cmake \
                -DCMAKE_BUILD_TYPE=${{ matrix.build_type }} \
                -S ${{ env.OPENVINO_TOKENIZERS_REPO }} \
                -B ${{ env.BUILD_DIR }}

      - name: Cmake build - tokenizers
        run: cmake --build ${{ env.BUILD_DIR }} --parallel --config ${{ matrix.build_type }} --verbose
        env:
          CMAKE_BUILD_PARALLEL_LEVEL: '4'

      - name: Cmake install - tokenizers
        run: |
          cmake --install ${{ env.BUILD_DIR }} --config=${{ matrix.build_type }} --prefix=${{ env.OV_TOKENIZERS_INSTALL_DIR }}/ov_tokenizers

      - name: Pack Artifacts
        run: |
          pushd ${OV_TOKENIZERS_INSTALL_DIR}
            tar -czvf ${BUILD_DIR}/ov_tokenizers.tar.gz *
          popd
          
      #
      # Upload build artifacts
      #

      - name: Upload openvino tokenizers package
        if: ${{ always() }}
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: openvino_tokenizers_cpack_${{ matrix.build_type }}
          path: ${{ env.BUILD_DIR }}/*.tar.gz
          if-no-files-found: 'error'

      - name: Upload manifest
        if: ${{ always() }}
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: manifest_${{ matrix.build_type }}
          path: ${{ env.MANIFEST_PATH }}
          if-no-files-found: 'error'

  openvino_tokenizers_wheel:
    name: OpenVINO tokenizers extension wheel
    needs: [ openvino_download ]
    if: |
      always() &&
      (needs.openvino_download.outputs.status == 'success')
    timeout-minutes: 25
    defaults:
      run:
        shell: bash
    runs-on: aks-linux-4-cores-16gb
    container:
      image: openvinogithubactions.azurecr.io/ov_build/ubuntu_22_04_x64:${{ needs.openvino_download.outputs.docker_tag }}
      volumes:
        - /mount:/mount
        - ${{ github.workspace }}:${{ github.workspace }}
      options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING
    env:
      OPENVINO_REPO: ${{ github.workspace }}/openvino
      INSTALL_DIR: ${{ github.workspace }}/openvino/install
      OPENVINO_TOKENIZERS_REPO: ${{ github.workspace }}/openvino_tokenizers
      BUILD_DIR: ${{ github.workspace }}/openvino_tokenizers/build

    steps:
      - name: Clone Openvino tokenizers
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          path: ${{ env.OPENVINO_TOKENIZERS_REPO }}
          
      - name: Set CI environment
        id: create_manifest
        uses: openvinotoolkit/openvino/.github/actions/create_manifest@master
        with:
          repos: ${{ env.OPENVINO_TOKENIZERS_REPO }}
          product_type: ${{ env.BASE_PRODUCT_TYPE }}_Release
          target_arch: 'x86_64'
          build_type: Release
          save_to: ${{ github.workspace }}

      - name: Download OpenVINO package
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: ${{ needs.openvino_download.outputs.ov_artifact_name }}
          path: ${{ env.INSTALL_DIR }}
          merge-multiple: true

      #
      # Build
      #

      - name: Build tokenizers wheel
        run: |
          python -m pip wheel -v --no-deps --wheel-dir ${BUILD_DIR} \
              --config-settings='override=wheel.build_tag="${{ github.run_number }}"' \
              --config-settings=override=cross.arch="manylinux_2_31_x86_64" \
              ${{ needs.openvino_download.outputs.ov_wheel_source }} \
              ${OPENVINO_TOKENIZERS_REPO}
        working-directory: ${{ env.INSTALL_DIR }}
        env:
          CMAKE_GENERATOR: 'Unix Makefiles'
          OpenVINODeveloperPackage_DIR: ${{ env.INSTALL_DIR }}/developer_package/cmake

      #
      # Upload build artifacts
      #

      - name: Upload openvino tokenizers wheel
        if: ${{ always() }}
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: openvino_tokenizers_wheel
          path: ${{ env.BUILD_DIR }}/*.whl
          if-no-files-found: 'error'

  openvino_tokenizers_tests:
    name: OpenVINO tokenizers tests
    needs: [ openvino_download, openvino_tokenizers_wheel]
    if: always() && needs.openvino_tokenizers_wheel.result == 'success'
    timeout-minutes: 65
    defaults:
      run:
        shell: bash
    runs-on: ubuntu-22.04
    env:
      INSTALL_DIR: ${{ github.workspace }}/openvino/install

    steps:
      - name: Clone Openvino tokenizers sources and tests
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Setup Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip'

      - name: Download tokenizers package
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: openvino_tokenizers_wheel
          path: ${{ env.INSTALL_DIR }}/ov_tokenizers

      - name: Download OpenVINO package
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: ${{ needs.openvino_download.outputs.ov_artifact_name }}
          path: ${{ env.INSTALL_DIR }}
          merge-multiple: true
          
      - name: Configure poetry
        uses: ./.github/actions/install_poetry
      
      - name: Find OpenVINO wheel
        uses: ./.github/actions/find_wheel
        id: ov_wheel
        with:
          wheels_dir: '${{ env.INSTALL_DIR }}/wheels'
          package_name: 'openvino'
             
      - name: Install OpenVINO wheel
        run: poetry add ${{ steps.ov_wheel.outputs.wheel_path }}
        
      - name: Find Tokenizers wheel
        uses: ./.github/actions/find_wheel
        id: tokenizers_wheel
        with:
          wheels_dir: '${{ env.INSTALL_DIR }}/ov_tokenizers'
          package_name: 'openvino_tokenizers'

      - name: Install OpenVINO tokenizers wheel
        run: poetry run pip install --no-deps ${{ steps.tokenizers_wheel.outputs.wheel_path }}
          
      - name: Install Test dependencies
        run: poetry install --extras=transformers --with dev,benchmark
        
      - name: Tokenizers regression tests (using openvino python wheels)
        run: poetry run pytest tests

      - name: Install transformers v4
        run: |
          poetry add "transformers@^4.0.0[sentencepiece]"

      - name: Tokenizers regression tests (with transformers v4)
        run: poetry run pytest tests
        
  openvino_tf_tests:
    name: OpenVINO TensorFlow tests
    needs: [ openvino_download, openvino_tokenizers_wheel]
    if: always() && needs.openvino_tokenizers_wheel.result == 'success'
    timeout-minutes: 45
    defaults:
      run:
        shell: bash
    runs-on: ubuntu-22.04
    env:
      OPENVINO_REPO: ${{ github.workspace }}/openvino
      INSTALL_DIR: ${{ github.workspace }}/openvino/install

    steps:
      - name: Clone tokenizers sources
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          sparse-checkout: |
            .github/actions
      
      - name: Clone Openvino TensorFlow tests
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
        with:
          repository: 'openvinotoolkit/openvino'
          ref: ${{ env.OV_BRANCH }}
          path: ${{ env.OPENVINO_REPO }}
          sparse-checkout: |
            tests/layer_tests/
            tests/requirements_tensorflow

      - name: Setup Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          cache: 'pip'

      - name: Download tokenizers package
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: openvino_tokenizers_wheel
          path: ${{ env.INSTALL_DIR }}/ov_tokenizers

      - name: Download OpenVINO package
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: ${{ needs.openvino_download.outputs.ov_artifact_name }}
          path: ${{ env.INSTALL_DIR }}
          merge-multiple: true

      - name: Configure poetry
        uses: ./.github/actions/install_poetry
      
      - name: Find OpenVINO wheel
        uses: ./.github/actions/find_wheel
        id: ov_wheel
        with:
          wheels_dir: '${{ env.INSTALL_DIR }}/wheels'
          package_name: 'openvino'
              
      - name: Install OpenVINO wheel
        run: poetry add ${{ steps.ov_wheel.outputs.wheel_path }}

      - name: Find Tokenizers wheel
        uses: ./.github/actions/find_wheel
        id: tokenizers_wheel
        with:
          wheels_dir: '${{ env.INSTALL_DIR }}/ov_tokenizers'
          package_name: 'openvino_tokenizers'

      - name: Install OpenVINO tokenizers wheel
        run: poetry run pip install --no-deps ${{ steps.tokenizers_wheel.outputs.wheel_path }}
          
      - name: Install Test dependencies
        run: poetry install --extras=transformers --with dev

      - name: Install Tensorflow tests dependencies
        run: poetry run pip install -r ${{ env.OPENVINO_REPO }}/tests/requirements_tensorflow

      - name: TensorFlow 1 Layer Tests
        run: poetry run pytest ${{ env.OPENVINO_REPO }}/tests/layer_tests/tensorflow_tests/ -n logical -m precommit  --junitxml=${{ env.OPENVINO_REPO }}/TEST-tf_fe.xml
        env:
          TEST_DEVICE: CPU
          TEST_PRECISION: FP16

      - name: TensorFlow 2 Layer Tests
        run: poetry run pytest ${{ env.OPENVINO_REPO }}/tests/layer_tests/tensorflow2_keras_tests/ -n logical -m precommit --junitxml=${{ env.OPENVINO_REPO }}/TEST-tf2_fe.xml
        env:
          TEST_DEVICE: CPU
          TEST_PRECISION: FP16

      - name: Upload Test Results
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        if: ${{ !cancelled() }}
        with:
          name: test-results-python-layers
          path: |
            ${{ env.OPENVINO_REPO }}/TEST*.html
            ${{ env.OPENVINO_REPO }}/TEST*.xml
          if-no-files-found: 'warn'


  store_artifacts:
    name: Store build artifacts
    strategy:
      matrix:
        build_type: [Release] # TODO: Add Debug build when OV provider is ready or use OV package
    needs: [openvino_download, openvino_tokenizers_wheel, openvino_tokenizers_cpack]
    timeout-minutes: 10
    defaults:
      run:
        shell: bash
    runs-on: aks-linux-medium
    container:
      image: openvinogithubactions.azurecr.io/library/python:3.12-slim
      volumes:
        - /mount:/mount
        - ${{ github.workspace }}:${{ github.workspace }}
    env:
      CPACK_PACKAGE: ${{ github.workspace }}/ov_tokenizers.tar.gz
      WHEEL_PACKAGE: ${{ github.workspace }}/wheels
      MANIFEST_PATH: ${{ github.workspace }}/manifest.yml

    steps:
      - name: Download tokenizers package & manifest
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          pattern: "{openvino_tokenizers_cpack_${{ matrix.build_type }},manifest_${{ matrix.build_type }}}"
          path: ${{ github.workspace }}
          merge-multiple: true

      - name: Download tokenizers wheels
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          name: openvino_tokenizers_wheel
          path: ${{ env.WHEEL_PACKAGE }}

      - name: Store ${{ matrix.build_type }} artifacts to a shared drive
        id: store_artifacts
        if: ${{ always() }}
        uses: openvinotoolkit/openvino/.github/actions/store_artifacts@master
        with:
          artifacts: |
            ${{ env.CPACK_PACKAGE }}
            ${{ env.WHEEL_PACKAGE }}
            ${{ env.MANIFEST_PATH }}
          storage_dir: ${{ env.BASE_PRODUCT_TYPE }}_${{ matrix.build_type }}
          storage_root: ${{ env.ARTIFACTS_SHARE }}
          product_name: ${{ github.event.repository.name }}


  Overall_Status:
    name: ci/gha_overall_status_linux
    needs: [openvino_download, openvino_tokenizers_cpack, openvino_tokenizers_wheel, openvino_tokenizers_tests, openvino_tf_tests]
    if: ${{ always() }}
    runs-on: ubuntu-latest
    steps:
      - name: Check status of all jobs
        if: >-
          ${{
            contains(needs.*.result, 'failure') ||
            contains(needs.*.result, 'cancelled')
          }}
        run: exit 1