feat: add AI utils methods to compare texts

Author: rgonalo

.github/workflows/ci.yml

@@ -4,7 +4,6 @@ on: [push, pull_request]
 
 jobs:
   test:
-
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -22,10 +21,15 @@ jobs:
         python -m pip install --upgrade pip
         pip install -r requirements.txt
         pip install -r requirements_dev.txt
+        python -m spacy download en_core_web_sm
     - name: Lint with flake8
       run: |
         flake8 . --count --max-complexity=10 --max-line-length=121 --show-source --statistics
     - name: Test with pytest and coverage
+      env:
+        AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+        AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
+        OPENAI_API_VERSION: ${{ secrets.OPENAI_API_VERSION }}
       run: |
         coverage run --source=toolium -m pytest toolium/test
     - name: Publish on coveralls.io

CHANGELOG.rst

@@ -6,6 +6,11 @@ v3.6.0
 
 *Release date: In development*
 
+- Add text comparison methods based on AI utils. To use them, install the `ai` extra dependency:
+
+.. code:: console
+
+    $ pip install toolium[ai]
 
 v3.5.0
 ------

docs/ai_utils.rst

@@ -0,0 +1,48 @@
+.. _tests_result_analysis:
+
+AI Utils
+========
+
+Text Similarity
+---------------
+
+Toolium provides several methods to compare text similarity using different techniques:
+
+1. **SpaCy**: Uses the SpaCy library to compute text similarity with SLM models.
+2. **Sentence Transformers**: Leverages Sentence Transformers for semantic textual similarity.
+3. **OpenAI LLM**: Utilizes OpenAI's language model for advanced semantic text comparison.
+
+You can use the function `check_text_similarity` from `toolium.utils.ai_utils` module to compare two texts using any of these
+methods. You can specify the method to use with the `similarity_method` parameter and set a threshold for similarity with the
+`threshold` parameter.
+
+.. code-block:: python
+
+    from toolium.utils.ai_utils import check_text_similarity
+
+    input_text = "Your input text here"
+    expected_text = "Your expected text here"
+    threshold = 0.8  # Similarity threshold between 0 and 1
+    similarity_method = 'spacy'  # Options: 'spacy', 'sentence_transformers', 'openai_llm'
+
+    check_text_similarity(input_text, expected_text, threshold=threshold, similarity_method=similarity_method)
+
+
+Default similarity method can be set in the properties.cfg file with the property
+`text_similarity_method <https://toolium.readthedocs.io/en/latest/ai_utils.html#configuration>`_ in *[AI]* section ::
+
+    [AI]
+    text_similarity_method: spacy
+
+Make sure to install the required libraries for the chosen method. For SpaCy, Sentence Transformers or OpenAI LLM, you
+can install them with the following command:
+
+.. code-block:: bash
+
+    pip install toolium[ai]
+
+For SpaCy, you also need to download the language model:
+
+.. code-block:: bash
+
+    python -m spacy download en_core_web_sm

docs/index.rst

@@ -12,6 +12,7 @@ Main Features
 - :ref:`Choosing driver through a configuration file <driver_configuration>`
 - :ref:`Page Object pattern <page_objects>`
 - :ref:`BDD integration <bdd_integration>`
+- :ref:`AI utils <ai_utils>`
 - :ref:`Visual testing solution <visual_testing>`
 - :ref:`Tests result analysis <tests_result_analysis>`
 
@@ -24,6 +25,7 @@ Library Reference
    driver_configuration.rst
    page_objects.rst
    bdd_integration.rst
+   ai_utils.rst
    visual_testing.rst
    tests_result_analysis.rst
    Changelog <changelog.rst>

docs/toolium.utils.rst

@@ -3,6 +3,16 @@
 utils
 =====
 
+.. _ai_utils:
+
+ai_utils
+--------
+
+.. automodule:: toolium.utils.ai_utils
+    :members:
+    :undoc-members:
+    :show-inheritance:
+
 .. _dataset:
 
 dataset

requirements_dev.txt

@@ -10,3 +10,7 @@ wheel~=0.40
 twine~=4.0
 behave==1.2.6  # behave tests
 importlib_metadata==7.2.1
+spacy~=3.8
+numpy~=1.26  # fix spaCy dependency error
+sentence-transformers~=5.1
+openai~=1.108

setup.py

@@ -63,6 +63,12 @@ def get_long_description():
     tests_require=read_file('requirements_dev.txt').splitlines(),
     extras_require={
         "playwright": ["playwright~=1.43"],
+        "ai": [
+            "spacy~=3.8",
+            "numpy~=1.26",  # fix spaCy dependency error
+            "sentence-transformers~=5.1",
+            "openai~=1.108"
+        ]
     },
     test_suite='toolium.test',
     author='Rubén González Alonso, Telefónica I+D',

toolium/test/conf/properties.cfg

@@ -58,3 +58,6 @@ fixversion:
 labels:
 comments:
 build:
+
+[AI]
+text_similarity_method: spacy

toolium/test/utils/test_ai_utils.py

@@ -0,0 +1,156 @@
+# -*- coding: utf-8 -*-
+"""
+Copyright 2025 Telefónica Innovación Digital, S.L.
+This file is part of Toolium.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import mock
+import pytest
+
+from toolium.driver_wrappers_pool import DriverWrappersPool
+from toolium.utils.ai_utils import (get_text_similarity_with_spacy, get_text_similarity_with_sentence_transformers,
+                                    get_text_similarity_with_openai_llm, check_text_similarity)
+
+
+get_similarity_examples = (
+    ('Today it will be sunny', 'Today it will be sunny', 0.9, 1),
+    ('Today is sunny', 'Today it will be sunny', 0.6, 0.9),
+    ('It is sunny', 'Today it will be sunny', 0.5, 0.7),
+    ('Nothing related', 'Today it will be sunny', 0, 0.6),
+)
+
+
+@pytest.mark.parametrize('input_text, expected_text, expected_low, expected_high', get_similarity_examples)
+def test_get_text_similarity_with_spacy(input_text, expected_text, expected_low, expected_high):
+    similarity = get_text_similarity_with_spacy(input_text, expected_text)
+    assert expected_low <= similarity <= expected_high
+
+
+@pytest.mark.parametrize('input_text, expected_text, expected_low, expected_high', get_similarity_examples)
+def test_get_text_similarity_with_sentence_transformers(input_text, expected_text, expected_low, expected_high):
+    similarity = get_text_similarity_with_sentence_transformers(input_text, expected_text)
+    assert expected_low <= similarity <= expected_high
+
+
+get_openai_similarity_examples = (
+    ('Today it will be sunny', 'Today it will be sunny', 0.9, 1),
+    ('Today is sunny', 'Today it will be sunny', 0.7, 0.9),
+    ('It is sunny', 'Today it will be sunny', 0.7, 0.9),
+    ('Nothing related', 'Today it will be sunny', 0, 0.1),
+    ('Today it will be cloudy', 'Today it will be sunny', 0, 0.1),
+    ('A splendid and rainless day is expected', 'Today it will be sunny', 0.8, 1),
+)
+
+
+@pytest.mark.parametrize('input_text, expected_text, expected_low, expected_high', get_openai_similarity_examples)
+def test_get_text_similarity_with_openai_llm(input_text, expected_text, expected_low, expected_high):
+    similarity = get_text_similarity_with_openai_llm(input_text, expected_text)
+    assert expected_low <= similarity <= expected_high
+
+
+check_similarity_passed_examples = (
+    ('Today it will be sunny', 'Today it will be sunny', 0.9),
+    ('Today it will be sunny', ['Nothing related', 'Today it will be sunny'], 0.9),
+    ('Today is sunny', ['It will be nice', 'Today it will be sunny'], 0.6),
+)
+
+
+@pytest.mark.parametrize('input_text, expected_text, threshold', check_similarity_passed_examples)
+def test_check_text_similarity_with_spacy_passed(input_text, expected_text, threshold):
+    check_text_similarity(input_text, expected_text, threshold=threshold, similarity_method='spacy')
+
+
+@pytest.mark.parametrize('input_text, expected_text, threshold', check_similarity_passed_examples)
+def test_check_text_similarity_with_sentence_transformers_passed(input_text, expected_text, threshold):
+    check_text_similarity(input_text, expected_text, threshold=threshold, similarity_method='sentence_transformers')
+
+
+@pytest.mark.parametrize('input_text, expected_text, threshold', check_similarity_passed_examples)
+def test_check_text_similarity_with_openai_llm_passed(input_text, expected_text, threshold):
+    check_text_similarity(input_text, expected_text, threshold=threshold, similarity_method='openai_llm')
+
+
+check_similarity_failed_examples = (
+    ('Today it will be sunny', 'Nothing related', 0.9),
+    ('Today it will be sunny', ['Nothing related', 'It is sunny'], 0.9),
+)
+
+
+@pytest.mark.parametrize('input_text, expected_text, threshold', check_similarity_failed_examples)
+def test_check_text_similarity_with_spacy_failed(input_text, expected_text, threshold):
+    with pytest.raises(Exception) as excinfo:
+        check_text_similarity(input_text, expected_text, threshold=threshold, similarity_method='spacy')
+    assert str(excinfo.value).startswith('Similarity between received and expected texts is below threshold')
+
+
+@pytest.mark.parametrize('input_text, expected_text, threshold', check_similarity_failed_examples)
+def test_check_text_similarity_with_sentence_transformers_failed(input_text, expected_text, threshold):
+    with pytest.raises(Exception) as excinfo:
+        check_text_similarity(input_text, expected_text, threshold=threshold, similarity_method='sentence_transformers')
+    assert str(excinfo.value).startswith('Similarity between received and expected texts is below threshold')
+
+
+check_openai_similarity_failed_examples = (
+    ('Today it will be sunny', 'Nothing related', 0.9),
+    ('Today it will be sunny', ['Nothing related', 'Today it is cold'], 0.9),
+)
+
+
+@pytest.mark.parametrize('input_text, expected_text, threshold', check_openai_similarity_failed_examples)
+def test_check_text_similarity_with_openai_llm_failed(input_text, expected_text, threshold):
+    with pytest.raises(Exception) as excinfo:
+        check_text_similarity(input_text, expected_text, threshold=threshold, similarity_method='openai_llm')
+    assert str(excinfo.value).startswith('Similarity between received and expected texts is below threshold')
+
+
+@mock.patch('toolium.utils.ai_utils.get_text_similarity_with_spacy')
+def test_check_text_similarity_with_default_method(similarity_mock):
+    similarity_mock.return_value = 0.9
+    input_text = 'Today it will be sunny'
+    expected_text = 'Today is sunny'
+    check_text_similarity(input_text, expected_text, threshold=0.8)
+    similarity_mock.assert_called_once_with(input_text, expected_text)
+
+
+@mock.patch('toolium.utils.ai_utils.get_text_similarity_with_sentence_transformers')
+def test_check_text_similarity_with_configured_method(similarity_mock):
+    config = DriverWrappersPool.get_default_wrapper().config
+    try:
+        config.add_section('AI')
+    except Exception:
+        pass
+    config.set('AI', 'text_similarity_method', 'sentence_transformers')
+    similarity_mock.return_value = 0.9
+
+    input_text = 'Today it will be sunny'
+    expected_text = 'Today is sunny'
+    check_text_similarity(input_text, expected_text, threshold=0.8)
+    similarity_mock.assert_called_once_with(input_text, expected_text)
+
+
+@mock.patch('toolium.utils.ai_utils.get_text_similarity_with_spacy')
+def test_check_text_similarity_with_configured_and_explicit_method(similarity_mock):
+    config = DriverWrappersPool.get_default_wrapper().config
+    try:
+        config.add_section('AI')
+    except Exception:
+        pass
+    config.set('AI', 'text_similarity_method', 'sentence_transformers')
+    similarity_mock.return_value = 0.9
+
+    input_text = 'Today it will be sunny'
+    expected_text = 'Today is sunny'
+    check_text_similarity(input_text, expected_text, threshold=0.8, similarity_method='spacy')
+    similarity_mock.assert_called_once_with(input_text, expected_text)