TF prepare_gradient_checkpointing, fix for newer TF #5690

Workflow file for this run

	name: CI

	on:
	push:
	branches:
	- master
	- main
	pull_request:
	workflow_dispatch:

	jobs:
	pycharm-inspect:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	python-version:
	- "3.10"
	torch-version:
	# Some code uses newer features.
	- 2.0.0
	tf-version:
	- 2.10.0

	steps:
	- uses: actions/checkout@v3

	- uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Cache dependencies
	uses: actions/cache@v2
	with:
	path: \|
	~/.cache
	~/.local
	key: cache-${{ runner.os }}-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-tf${{ matrix.tf-version }}
	restore-keys: \|
	cache-${{ runner.os }}-py${{ matrix.python-version }}-

	- name: Setup dependencies
	run: \|
	echo "PATH=$PATH:~/.local/bin" >> $GITHUB_ENV
	echo "Python ${{matrix.python-version}}"
	echo "pip: $(type pip)"

	pip install --user --upgrade pip setuptools wheel
	pip install --user typing # needed for Python 2 in some cases

	pip install --user pytest
	pip install --user nose

	pip install --user dm-tree h5py typing
	pip install --user --progress-bar=off "numpy<2"
	pip install --user --progress-bar=off scipy # for some tests

	pip install --user --upgrade --progress-bar=off torch==${{ matrix.torch-version }}

	# Retry several times in case download breaks. https://github.com/pypa/pip/issues/4796
	for ((i=0; i<3; i++)); do
	if pip install --user --upgrade --progress-bar=off tensorflow==${{ matrix.tf-version }}; then break; fi
	done

	# Used in some tests.
	pip install --user --progress-bar=off transformers
	pip install --user --progress-bar=off espnet

	- name: Test Python/Numpy/TF versions.
	run: \|
	echo "Python env: $(type python) $(python --version)"
	python -c "import numpy; print('NumPy:', numpy.version.full_version)"
	python -c "import torch; print('PyTorch:', torch.__version__, torch.__file__)"
	# torch.__version__ is for example "1.13.1+cu117"
	python -c "import torch; assert (torch.__version__ + '+').startswith('${{ matrix.torch-version }}+')"
	python -c "import tensorflow as tf; print('TensorFlow:', tf.__git_version__, tf.__version__, tf.__file__)"
	python -c "import tensorflow as tf; assert tf.__version__ == '${{ matrix.tf-version }}'"

	- name: Cache PyCharm
	uses: actions/cache@v2
	with:
	path: \|
	~/.cache/JetBrains
	key: cache-pycharm-${{ runner.os }}-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-tf${{ matrix.tf-version }}

	- name: Run PyCharm inspect
	run: \|
	tests/pycharm-inspect.py

	base-tests:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	python-version:
	- 3.7
	action:
	- TEST=Dataset
	- TEST=fork_exec
	- TEST=GeneratingDataset
	- TEST=hdf_dump
	- TEST=HDFDataset
	- TEST=LearningRateControl
	- TEST=MultiProcDataset
	- TEST=Pretrain
	- TEST=SprintDataset
	- TEST=TaskSystem
	- TEST=TaskSystem_SharedMem
	- TEST=tensor
	- TEST=TranslationDataset
	- TEST=Util

	steps:
	- uses: actions/checkout@v3

	- uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Cache dependencies
	uses: actions/cache@v2
	with:
	path: \|
	~/.cache
	~/.local
	key: cache-${{ runner.os }}-py${{ matrix.python-version }}-no-tf
	restore-keys: \|
	cache-${{ runner.os }}-py${{ matrix.python-version }}-

	- name: Setup dependencies
	run: \|
	echo "PATH=$PATH:~/.local/bin" >> $GITHUB_ENV
	echo "Python ${{matrix.python-version}}"
	echo "pip: $(type pip)"

	pip install --user --upgrade pip setuptools wheel
	pip install --user typing # needed for Python 2 in some cases

	pip install --user pytest

	# Install nose anyway because we currently use some of its helper functions.
	pip install --user nose

	pip install --user --progress-bar=off -r requirements.txt

	- name: Test Python/Numpy/TF versions.
	run: \|
	echo "Python env: $(type python) $(python --version)"
	python -c "import numpy; print('NumPy:', numpy.version.full_version)"

	echo "Pytest env:"
	python -m pytest -V

	- name: Run test
	run: \|
	export ${{ matrix.action }}
	python -m pytest tests/test_$TEST.py

	tf-tests:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	python-version:
	- 3.7
	tf-version:
	- 2.10.0
	action:
	- TEST=Config
	- TEST=Fsa
	- TEST=Log
	- TEST=SprintInterface
	- TEST=TFEngine
	- TEST=TFNativeOp
	- TEST=TFNetworkLayer
	- TEST=TFNetworkRecLayer
	- TEST=TFUpdater
	- TEST=TFUtil
	- TEST=tools
	include:
	- action: TEST=TFUtil
	python-version: 3.7
	tf-version: 1.15.3
	- action: TEST=TFEngine
	python-version: 3.7
	tf-version: 1.15.3
	- action: TEST=TFEngine
	python-version: 3.7
	tf-version: 2.3.0
	- action: TEST=TFNativeOp
	python-version: 3.7
	tf-version: 1.14.0
	- action: TEST=TFNetworkSigProcLayer
	python-version: 3.7
	tf-version: 1.15.3
	- action: TEST=demos
	python-version: 3.7
	tf-version: 2.3.0

	steps:
	- uses: actions/checkout@v3

	- uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Cache dependencies
	uses: actions/cache@v2
	with:
	path: \|
	~/.cache
	~/.local
	key: cache-${{ runner.os }}-py${{ matrix.python-version }}-tf${{ matrix.tf-version }}
	restore-keys: \|
	cache-${{ runner.os }}-py${{ matrix.python-version }}-

	- name: Setup dependencies
	run: \|
	echo "PATH=$PATH:~/.local/bin" >> $GITHUB_ENV
	echo "Python ${{matrix.python-version}}"
	echo "pip: $(type pip)"

	pip install --user --upgrade pip setuptools wheel
	pip install --user typing # needed for Python 2 in some cases

	pip install --user pytest

	# Install nose anyway because we currently use some of its helper functions.
	pip install --user nose

	if [[ "${{matrix.tf-version}}" == 2.[0123].* \|\| "${{matrix.tf-version}}" == 1.* ]]; then
	# Older TF needs older NumPy version.
	# https://github.com/rwth-i6/returnn/pull/1160#issuecomment-1284537803
	pip install --user numpy==1.19.5
	# Older TF needs also older protobuf version.
	# https://github.com/rwth-i6/returnn/issues/1209
	pip install --user "protobuf<=3.20.1"
	fi

	if [[ "${{matrix.tf-version}}" == 2.10.* ]]; then
	# TF 2.10 requires gast<=0.4.0,>=0.2.1. But for example, with gast 0.2.2, we get some AutoGraph error:
	# Cause: module 'gast' has no attribute 'Constant'
	# Similar like: https://github.com/tensorflow/tensorflow/issues/47802
	pip install --user --upgrade "gast<=0.4.0"
	fi

	pip install --user --progress-bar=off -r requirements.txt
	pip install --user --progress-bar=off scipy # for some tests
	pip install --user --upgrade --progress-bar=off tensorflow==${{ matrix.tf-version }}

	- name: Test Python/Numpy/TF versions.
	run: \|
	echo "Python env: $(type python) $(python --version)"
	python -c "import numpy; print('NumPy:', numpy.version.full_version)"
	python -c "import tensorflow as tf; print('TensorFlow:', tf.__git_version__, tf.__version__, tf.__file__)"
	python -c "import tensorflow as tf; assert tf.__version__ == '${{ matrix.tf-version }}'"

	echo "Pytest env:"
	python -m pytest -V

	- name: Run test
	run: \|
	export ${{ matrix.action }}
	python -m pytest tests/test_$TEST.py

	torch-tests:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	python-version:
	- "3.10"
	torch-version:
	- 1.13.1
	- 2.0.0
	action:
	- TEST=demos RETURNN_DISABLE_TF=1
	- TEST=torch_dataset
	- TEST=torch_engine
	- TEST=torch_frontend
	- TEST=torch_internal_frontend
	- TEST=torch_util

	steps:
	- uses: actions/checkout@v3

	- uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Cache dependencies
	uses: actions/cache@v2
	with:
	path: \|
	~/.cache
	~/.local
	key: cache-${{ runner.os }}-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}
	restore-keys: \|
	cache-${{ runner.os }}-py${{ matrix.python-version }}-

	- name: Setup dependencies
	run: \|
	echo "PATH=$PATH:~/.local/bin" >> $GITHUB_ENV
	echo "Python ${{matrix.python-version}}"
	echo "pip: $(type pip)"

	pip install --user --upgrade pip setuptools wheel
	pip install --user typing # needed for Python 2 in some cases

	pip install --user pytest

	# Install nose anyway because we currently use some of its helper functions.
	pip install --user nose

	pip install --user dm-tree h5py typing
	pip install --user --progress-bar=off "numpy<2"
	pip install --user --progress-bar=off scipy # for some tests
	pip install --user --upgrade --progress-bar=off torch==${{ matrix.torch-version }}
	pip install --user --progress-bar=off onnx onnxruntime
	pip install --user --progress-bar=off lovely_tensors

	- name: Test Python/Numpy/PyTorch versions.
	run: \|
	echo "Python env: $(type python) $(python --version)"
	python -c "import numpy; print('NumPy:', numpy.version.full_version)"
	python -c "import torch; print('PyTorch:', torch.__version__, torch.__file__)"
	# torch.__version__ is for example "1.13.1+cu117"
	python -c "import torch; assert (torch.__version__ + '+').startswith('${{ matrix.torch-version }}+')"

	echo "Pytest env:"
	python -m pytest -V

	- name: Run test
	run: \|
	export ${{ matrix.action }}
	python -m pytest tests/test_$TEST.py

	rf-tests:
	runs-on: ubuntu-latest
	strategy:
	fail-fast: false
	matrix:
	python-version:
	- "3.10"
	torch-version:
	- 2.0.0
	tf-version:
	- 2.10.0
	action:
	- TEST=rf_array
	- TEST=rf_attention
	- TEST=rf_base
	- TEST=rf_cond
	- TEST=rf_const
	- TEST=rf_container
	- TEST=rf_conv
	- TEST=rf_decoder_transformer
	- TEST=rf_encoder_conformer
	- TEST=rf_gradient
	- TEST=rf_label_smoothing
	- TEST=rf_loop
	- TEST=rf_math
	- TEST=rf_normalization
	- TEST=rf_piecewise_linear
	- TEST=rf_rec
	- TEST=rf_reduce
	- TEST=rf_signal
	include:
	# Some selected tests (but not all) with some other versions.
	- action: TEST=rf_base
	python-version: 3.7
	torch-version: 1.13.1
	tf-version: 2.10.0

	steps:
	- uses: actions/checkout@v3

	- uses: actions/setup-python@v4
	with:
	python-version: ${{ matrix.python-version }}

	- name: Cache dependencies
	uses: actions/cache@v2
	with:
	path: \|
	~/.cache
	~/.local
	key: cache-${{ runner.os }}-py${{ matrix.python-version }}-torch${{ matrix.torch-version }}-tf${{ matrix.tf-version }}
	restore-keys: \|
	cache-${{ runner.os }}-py${{ matrix.python-version }}-

	- name: Setup dependencies
	run: \|
	echo "PATH=$PATH:~/.local/bin" >> $GITHUB_ENV
	echo "Python ${{matrix.python-version}}"
	echo "pip: $(type pip)"

	pip install --user --upgrade pip setuptools wheel
	pip install --user typing # needed for Python 2 in some cases

	pip install --user pytest

	# Install nose anyway because we currently use some of its helper functions.
	pip install --user nose

	pip install --user dm-tree h5py typing
	pip install --user --progress-bar=off "numpy<2"
	pip install --user --progress-bar=off scipy # for some tests

	pip install --user --upgrade torch==${{ matrix.torch-version }}
	pip install --user --progress-bar=off lovely_tensors

	if [[ "${{matrix.tf-version}}" == 2.10.* ]]; then
	# TF 2.10 requires gast<=0.4.0,>=0.2.1. But for example, with gast 0.2.2, we get some AutoGraph error:
	# Cause: module 'gast' has no attribute 'Constant'
	# Similar like: https://github.com/tensorflow/tensorflow/issues/47802
	pip install --user --upgrade "gast<=0.4.0"
	fi

	# Retry several times in case download breaks. https://github.com/pypa/pip/issues/4796
	for ((i=0; i<3; i++)); do
	if pip install --user --upgrade --progress-bar=off tensorflow==${{ matrix.tf-version }}; then break; fi
	done

	# Needed for some tests.
	pip install --user --progress-bar=off transformers
	pip install --user --progress-bar=off espnet
	# TorchAudio needed by ESPnet.
	# https://pytorch.org/audio/stable/installation.html#compatibility-matrix
	if [[ "${{matrix.torch-version}}" == 2.0.0 ]]; then
	pip install --user --progress-bar=off torchaudio==2.0.1
	elif [[ "${{matrix.torch-version}}" == 1.13.1 ]]; then
	pip install --user --progress-bar=off torchaudio==0.13.1
	else
	pip install --user --progress-bar=off torchaudio==${{matrix.torch-version}}
	fi

	- name: Test Python/Numpy/PyTorch versions.
	run: \|
	echo "Python env: $(type python) $(python --version)"
	python -c "import numpy; print('NumPy:', numpy.version.full_version)"
	python -c "import torch; print('PyTorch:', torch.__version__, torch.__file__)"
	# torch.__version__ is for example "1.13.1+cu117"
	python -c "import torch; assert (torch.__version__ + '+').startswith('${{ matrix.torch-version }}+')"
	python -c "import tensorflow as tf; print('TensorFlow:', tf.__git_version__, tf.__version__, tf.__file__)"
	python -c "import tensorflow as tf; assert tf.__version__ == '${{ matrix.tf-version }}'"

	echo "Pytest env:"
	python -m pytest -V

	- name: Run test
	run: \|
	export ${{ matrix.action }}
	python -m pytest tests/test_$TEST.py

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

TF prepare_gradient_checkpointing, fix for newer TF #5690

Workflow file

TF prepare_gradient_checkpointing, fix for newer TF #5690

Jobs

Run details

Workflow file for this run