version: 2.1

jobs:
  build:
    parameters:
      docker_image:
        type: string
        default: ""
      apt_get:
        type: string
        default: ""
      c_compiler:
        type: string
        default: ""
      cxx_compiler:
        type: string
        default: ""
      cmake_args:
        type: string
        default: ""
      nproc:
        type: integer
        default: 20
    docker:
      - image: << parameters.docker_image >>
    steps:
      - checkout
      - run:
          name: Install apt packages
          command: |
            apt-get update
            apt-get install -y git-core build-essential cmake << parameters.apt_get >>
      - run:
          name: Initialize submodules
          command: |
            git submodule init
            git submodule update
      - run:
          name: Build
          command: |
            mkdir build
            cd build
            cmake ../ \
              -DCMAKE_C_FLAGS="-Werror -Wno-deprecated-declarations" \
              -DCMAKE_CXX_FLAGS="-Werror -Wno-deprecated-declarations" \
              -DCMAKE_C_COMPILER=<< parameters.c_compiler >> \
              -DCMAKE_CXX_COMPILER=<< parameters.cxx_compiler >> \
              -DTP_ENABLE_CMA=OFF \
              -DTP_ENABLE_CUDA_IPC=OFF \
              -DTP_ENABLE_IBV=OFF \
              -DTP_BUILD_TESTING=ON \
              << parameters.cmake_args >>
            make -j<<parameters.nproc>>
      - run:
          name: Test
          command: |
            cd build
            ./tensorpipe/test/tensorpipe_test
      - run:
          name: Install
          command: |
            cd build
            make install
  build_gpu:
    parameters:
      cuda_version:
        type: string
      exclude_tests:
        type: string
        default: ""
    machine:
      resource_class: gpu.medium
      image: ubuntu-1604-cuda-10.1:201909-23
      docker_layer_caching: true
    steps:
      - checkout
      - run:
          name: Initialize submodules
          command: |
            git submodule init
            git submodule update
      - run:
          name: Build/test
          command: |
              docker build -t tensorpipe -f .circleci/Dockerfile.cuda<< parameters.cuda_version >> .
              docker run --gpus all --pid=host tensorpipe sh -c "
                mkdir build && cd build &&
                cmake ../ \
                  -DCMAKE_C_FLAGS=\"-Werror -Wno-deprecated-declarations\" \
                  -DCMAKE_CXX_FLAGS=\"-Werror -Wno-deprecated-declarations\" \
                  -DCUDA_NVCC_FLAGS=\"-gencode arch=compute_61,code=sm_61\" \
                  -DTP_ENABLE_SHM=OFF \
                  -DTP_ENABLE_CMA=OFF \
                  -DTP_USE_CUDA=ON \
                  -DTP_ENABLE_CUDA_IPC=ON \
                  -DTP_ENABLE_IBV=OFF \
                  -DTP_BUILD_TESTING=ON &&
                make -j20 &&
                ./tensorpipe/test/tensorpipe_test --gtest_filter='-<< parameters.exclude_tests >>' &&
                make install"
  bare_metal:
    parameters:
      image:
        type: string
        default: ""
      apt_get:
        type: string
        default: ""
      c_compiler:
        type: string
        default: ""
      cxx_compiler:
        type: string
        default: ""
      cmake_args:
        type: string
        default: ""
      nproc:
        type: integer
        default: 20
    machine:
      image: << parameters.image >>
    steps:
      - checkout
      - run:
          name: Install apt packages
          command: |
            sudo apt-get update
            sudo apt-get install -y git-core build-essential cmake libibverbs1 rdma-core linux-modules-extra-$(uname -r) << parameters.apt_get >>
      - run:
          name: Initialize submodules
          command: |
            git submodule init
            git submodule update
      - run:
          name: Build
          command: |
            mkdir build
            cd build
            cmake ../ \
              -DCMAKE_C_FLAGS="-Werror -Wno-deprecated-declarations" \
              -DCMAKE_CXX_FLAGS="-Werror -Wno-deprecated-declarations" \
              -DCMAKE_C_COMPILER=<< parameters.c_compiler >> \
              -DCMAKE_CXX_COMPILER=<< parameters.cxx_compiler >> \
              -DTP_ENABLE_CUDA_IPC=OFF \
              -DTP_ENABLE_IBV=ON \
              -DTP_BUILD_TESTING=ON \
              << parameters.cmake_args >>
            make -j<<parameters.nproc>>
      - run:
          name: Configure Soft-RoCE (RXE) InfiniBand interface
          command: |
            # Find the name of the first non-loopback IP interface
            INTERFACE_NAME=$(ip link | grep '^2: ' | sed -re 's/2: ([a-z0-9]+): .*/\1/')
            sudo rdma link add rxe0 type rxe netdev $INTERFACE_NAME
      - run:
          name: Test
          command: |
            cd build
            ./tensorpipe/test/tensorpipe_test
      - run:
          name: Test CMA channel autodetection with Docker
          command: |
            bash -eo pipefail tensorpipe/test/channel/cma/docker_tests.sh
      - run:
          name: Install
          command: |
            cd build
            sudo make install
  build_osx:
    macos:
      xcode: 12.4.0
    steps:
      - checkout
      - run:
          name: Install homebrew packages
          command: |
            brew install cmake
      - run:
          name: Initialize submodules
          command: |
            git submodule init
            git submodule update
      - run:
          name: Build
          command: |
            mkdir build
            cd build
            cmake ../ \
              -DCMAKE_C_FLAGS="-Werror -Wno-deprecated-declarations" \
              -DCMAKE_CXX_FLAGS="-Werror -Wno-deprecated-declarations" \
              -DTP_BUILD_TESTING=ON
            make -j
      - run:
          name: Test
          command: |
            cd build
            ./tensorpipe/test/tensorpipe_test
      - run:
          name: Install
          command: |
            cd build
            make install
  python:
    parameters:
      docker_image:
        type: string
        default: ""
      apt_get:
        type: string
        default: ""
    docker:
      - image: << parameters.docker_image >>
    steps:
      - checkout
      - run:
          name: Install apt packages
          command: |
            apt-get update
            apt-get install -y git-core build-essential cmake python3-dev python3-venv << parameters.apt_get >>
      - run:
          name: Initialize submodules
          command: |
            git submodule init
            git submodule update
      - run:
          name: Build
          command: |
            python3 -m venv venv
            source venv/bin/activate
            TP_ENABLE_CMA=OFF TP_ENABLE_CUDA_IPC=OFF TP_ENABLE_IBV=OFF python3 setup.py install
      - run:
          name: Test
          command: |
            source venv/bin/activate
            python3 tensorpipe/test/python/tensorpipe.py
  format:
    docker:
      - image: ubuntu:18.04
    steps:
      - checkout
      - run:
          name: Install clang-format
          command: |
            apt-get update
            apt-get install -y git-core clang-format-10
      - run:
          name: Verify clang-format
          command: |
             git ls-files | grep -E  '\.(cc|h)$' | xargs clang-format-10 -i
             if git diff --quiet; then
               echo "Formatting OK!"
             else
               echo "Formatting not OK!"
               echo "------------------"
               git --no-pager diff --color
               exit 1
             fi

workflows:
  build:
    jobs:
      - build:
          name: gcc5
          docker_image: ubuntu:18.04
          apt_get: "gcc-5 g++-5"
          c_compiler: gcc-5
          cxx_compiler: g++-5
      - build:
          name: gcc7
          docker_image: ubuntu:18.04
          apt_get: "gcc-7 g++-7"
          c_compiler: gcc-7
          cxx_compiler: g++-7
      - build:
          name: clang6
          docker_image: ubuntu:18.04
          apt_get: "clang-6.0"
          c_compiler: clang-6.0
          cxx_compiler: clang++-6.0
      - build:
          name: gcc7-asan
          docker_image: ubuntu:18.04
          apt_get: "gcc-7 g++-7"
          c_compiler: gcc-7
          cxx_compiler: g++-7
          cmake_args: -DSANITIZE=address
      - build:
          name: gcc7-tsan
          docker_image: ubuntu:18.04
          apt_get: "gcc-7 g++-7"
          c_compiler: gcc-7
          cxx_compiler: g++-7
          cmake_args: -DSANITIZE=thread
      - bare_metal:
          name: bare-metal
          image: ubuntu-2004:202008-01
          apt_get: "gcc-7 g++-7"
          c_compiler: gcc-7
          cxx_compiler: g++-7
      - build_gpu:
          name: GPU (CUDA 9.2)
          cuda_version: "9.2"
          # Excluding CudaGdr for lack of InfiniBand hardware, and CudaIpc on
          # multi GPU for lack of p2p capabilities.
          exclude_tests: "CudaGdr*:CudaIpc/CudaMultiGPUChannelTestSuite*"
      - build_gpu:
          name: GPU (CUDA 10.1)
          cuda_version: "10.1"
          # Excluding CudaGdr for lack of InfiniBand hardware, and CudaIpc on
          # multi GPU for lack of p2p capabilities.
          exclude_tests: "CudaGdr*:CudaIpc/CudaMultiGPUChannelTestSuite*"
      - build_gpu:
          name: GPU (CUDA 10.2)
          cuda_version: "10.2"
          # Excluding CudaGdr for lack of InfiniBand hardware, and CudaIpc on
          # multi GPU for lack of p2p capabilities.
          exclude_tests: "CudaGdr*:CudaIpc/CudaMultiGPUChannelTestSuite*"
      - build_gpu:
          name: GPU (CUDA 11.0)
          cuda_version: "11.0"
          # Excluding CudaGdr for lack of InfiniBand hardware, and CudaIpc on
          # multi GPU for lack of p2p capabilities.
          exclude_tests: "CudaGdr*:CudaIpc/CudaMultiGPUChannelTestSuite*"
      - build_gpu:
          name: GPU (CUDA 11.1)
          cuda_version: "11.1"
          # Excluding CudaGdr for lack of InfiniBand hardware, and CudaIpc on
          # multi GPU for lack of p2p capabilities, and CudaBasic/CudaMultiGPUChannelTestSuite.SendAcrossNonDefaultDevices/0
          # because it does not work with CUDA 11.1 (cf. https://github.com/pytorch/tensorpipe/issues/368).
          exclude_tests: "CudaGdr*:CudaIpc/CudaMultiGPUChannelTestSuite*:CudaBasic/CudaMultiGPUChannelTestSuite.SendAcrossNonDefaultDevices/0"
      - build_osx:
          name: OSX
      - python:
          name: python
          docker_image: ubuntu:18.04
          apt_get: "clang-6.0"
      - format:
          name: clang-format
