Improvement to tokenizer, remove use_mask #590

	name: Run Tests

	on: [push, pull_request]

	jobs:
	unittests:
	runs-on: ubuntu-latest

	steps:
	- name: Check out code
	uses: actions/checkout@v4

	- name: Set up Python
	uses: actions/setup-python@v5
	with:
	python-version: '3.10'

	- name: Install system dependencies
	run: sudo apt-get update && sudo apt-get install -y libgl1-mesa-glx

	- name: Install dependencies
	run: \|
	python -m pip install --upgrade pip
	pip install -r requirements.txt

	- name: Test tokenizer
	run: \|
	python -m unittest tests/test_tokenizer.py
	env:
	TF_CPP_MIN_LOG_LEVEL: '2'

	- name: Test DataLoader
	if: always()
	run: \|
	python -m unittest tests/test_dataloader.py
	env:
	TF_CPP_MIN_LOG_LEVEL: '2'

	- name: Test data augment layers
	if: always()
	run: \|
	python -m unittest tests/test_data_augments.py
	env:
	TF_CPP_MIN_LOG_LEVEL: '2'

	- name: Test DataManager
	if: always()
	run: \|
	python -m unittest tests/test_datamanager.py
	env:
	TF_CPP_MIN_LOG_LEVEL: '2'

	- name: Test model creation
	if: always()
	run: \|
	python -m unittest tests/test_model_creation.py
	env:
	TF_CPP_MIN_LOG_LEVEL: '2'

	- name: Test model to VGSL
	if: always()
	run: \|
	python -m unittest tests/test_model_to_vgsl.py
	env:
	TF_CPP_MIN_LOG_LEVEL: '2'

	- name: Test replacing model layers
	if: always()
	run: \|
	python -m unittest tests/test_model_replacement.py
	env:
	TF_CPP_MIN_LOG_LEVEL: '2'

	- name: Test custom LR schedule
	if: always()
	run: \|
	python -m unittest tests/test_lr_schedule.py
	env:
	TF_CPP_MIN_LOG_LEVEL: '2'

Provide feedback