Bump to skyportal c90babec2dde0373afcd09d515e018e29cf36be9 and skyportal-fink-client to 79aeb16d96af2a0576354fdcaa60fe0ea88c7058

.github/workflows/test_api_and_frontend.yaml

@@ -6,6 +6,9 @@ on:
       - main
   pull_request:
 
+env:
+  FRONTEND_TEST_HEADLESS: "1"
+
 jobs:
   test:
     runs-on: ubuntu-latest
@@ -44,9 +47,9 @@ jobs:
         with:
           python-version: "3.10"
 
-      - uses: actions/setup-node@v3
+      - uses: actions/setup-node@v5
         with:
-          node-version: 20
+          node-version: "22.x"
 
       - name: Checkout main
         uses: actions/checkout@v4
@@ -66,6 +69,10 @@ jobs:
         run: |
           git submodule update --init --recursive
 
+      - uses: browser-actions/setup-geckodriver@latest
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
       - uses: actions/cache@v4
         with:
           path: |
@@ -87,16 +94,7 @@ jobs:
           sudo add-apt-repository ppa:mozillateam/ppa
           printf 'Package: *\nPin: release o=LP-PPA-mozillateam\nPin-Priority: 1001' | sudo tee /etc/apt/preferences.d/mozilla-firefox
 
-          sudo apt install -y wget unzip firefox libcurl4-gnutls-dev libgnutls28-dev
-
-          # if nginx is already installed, remove it
-          sudo apt remove -y nginx nginx-common nginx-core nginx-full
-          sudo apt purge -y nginx nginx-common nginx-core nginx-full
-
-          # add the PPA repository with brotli support for nginx
-          sudo add-apt-repository ppa:ondrej/nginx -y
-          sudo apt update -y
-          sudo apt install nginx libnginx-mod-http-brotli-static libnginx-mod-http-brotli-filter -y
+          sudo apt install -y wget unzip firefox libcurl4-gnutls-dev libgnutls28-dev nginx
 
           pip install pip==24.0
           pip install wheel numpy
@@ -178,24 +176,11 @@ jobs:
           pip list --format=columns
           npm ls --depth 0 || true
 
-      - name: Install Geckodriver / Selenium
+      - name: Verify Geckodriver / Selenium
         run: |
-          GECKO_VER=0.34.0
-          CACHED_DOWNLOAD_DIR=~/.local/downloads
-          FILENAME=geckodriver-v${GECKO_VER}-linux64.tar.gz
-
-          if [[ ! -f ${CACHED_DOWNLOAD_DIR=}/${FILENAME} ]]; then
-            wget https://github.com/mozilla/geckodriver/releases/download/v${GECKO_VER}/${FILENAME} --directory-prefix=${CACHED_DOWNLOAD_DIR} --no-clobber
-          fi
-          sudo tar -xzf ${CACHED_DOWNLOAD_DIR}/geckodriver-v${GECKO_VER}-linux64.tar.gz -C /usr/local/bin
           geckodriver --version
           python -c "import selenium; print(f'Selenium {selenium.__version__}')"
 
-      - name: Patch test_frontend.py for a longer timeout
-        run: |
-          cd patched_skyportal
-          sed -i 's/timeout=60/timeout=180/g' baselayer/tools/test_frontend.py
-
       - name: Patch Notifications.jsx for a longer display
         run: |
           cd patched_skyportal
@@ -212,39 +197,39 @@ jobs:
         if: ${{ matrix.test_subset == 'frontend_pt1' }}
         run: |
           cd patched_skyportal
-          PYTHONPATH=. python baselayer/tools/test_frontend.py --xml --headless '--ignore=skyportal/tests/frontend/sources_and_observingruns_etc/test_source_count.py skyportal/tests/frontend/sources_and_observingruns_etc'
+          PYTHONPATH=. python baselayer/tools/test_frontend.py --xml '--ignore=skyportal/tests/frontend/sources_and_observingruns_etc/test_source_count.py skyportal/tests/frontend/sources_and_observingruns_etc'
 
       - name: Run front-end tests part 2
         if: ${{ matrix.test_subset == 'frontend_pt2' }}
         run: |
           cd patched_skyportal
-          PYTHONPATH=. python baselayer/tools/test_frontend.py --xml --headless '--ignore=skyportal/tests/frontend/sources_and_observingruns_etc --ignore=skyportal/tests/frontend/test_weather.py --ignore=skyportal/tests/frontend/sources_and_observingruns_etc/test_source_count.py --ignore=skyportal/tests/frontend/test_remote.py skyportal/tests/frontend'
+          PYTHONPATH=. python baselayer/tools/test_frontend.py --xml '--ignore=skyportal/tests/frontend/sources_and_observingruns_etc --ignore=skyportal/tests/frontend/test_weather.py --ignore=skyportal/tests/frontend/sources_and_observingruns_etc/test_source_count.py --ignore=skyportal/tests/frontend/test_remote.py skyportal/tests/frontend'
 
       - name: Run API & utils tests part 1
         if: ${{ matrix.test_subset == 'api_and_utils_pt1' }}
         run: |
           cd patched_skyportal
-          PYTHONPATH=. python baselayer/tools/test_frontend.py --xml --headless 'skyportal/tests/api/candidates_sources_events'
+          PYTHONPATH=. python baselayer/tools/test_frontend.py --xml 'skyportal/tests/api_tests/candidates_sources_events'
 
       - name: Run API & utils tests part 2
         if: ${{ matrix.test_subset == 'api_and_utils_pt2' }}
         run: |
           cd patched_skyportal
-          PYTHONPATH=. python baselayer/tools/test_frontend.py --xml --headless '--ignore=skyportal/tests/api/candidates_sources_events skyportal/tests/api skyportal/tests/tools skyportal/tests/utils skyportal/tests/rate_limiting'
+          PYTHONPATH=. python baselayer/tools/test_frontend.py --xml '--ignore=skyportal/tests/api_tests/candidates_sources_events skyportal/tests/api_tests skyportal/tests/tools skyportal/tests/utils skyportal/tests/rate_limiting'
 
       - name: Run flaky tests
         if: ${{ matrix.test_subset == 'flaky' }}
         continue-on-error: true
         run: |
           cd patched_skyportal
-          PYTHONPATH=. python baselayer/tools/test_frontend.py --xml --headless 'skyportal/tests/flaky'
+          PYTHONPATH=. python baselayer/tools/test_frontend.py --xml 'skyportal/tests/flaky'
 
       - name: Run external tests
         if: ${{ matrix.test_subset == 'external' }}
         continue-on-error: true
         run: |
           cd patched_skyportal
-          PYTHONPATH=. python baselayer/tools/test_frontend.py --xml --headless 'skyportal/tests/external'
+          PYTHONPATH=. python baselayer/tools/test_frontend.py --xml 'skyportal/tests/external'
 
       - name: Upload logs
         uses: actions/upload-artifact@v4

extensions/skyportal/baselayer/app/psa.py

@@ -751,7 +751,7 @@ def ACCESS_TOKEN_URL(self):
         # Instead, we always connect to localhost:63000.
 
         env, cfg = load_env()
-        return f'http://localhost:{cfg["ports.fake_oauth"]}/fakeoauth2/token'
+        return f"http://localhost:{cfg['ports.fake_oauth']}/fakeoauth2/token"
 
     def user_data(self, access_token, *args, **kwargs):
         return {"id": "testuser@cesium-ml.org", "email": "testuser@cesium-ml.org"}

extensions/skyportal/skyportal/tests/test_util.py

@@ -0,0 +1,215 @@
+import os
+
+import pytest
+from baselayer.app import models
+from baselayer.app.config import load_config
+from selenium import webdriver
+from selenium.common.exceptions import (
+    ElementClickInterceptedException,
+    JavascriptException,
+    NoSuchElementException,
+    StaleElementReferenceException,
+    TimeoutException,
+)
+from selenium.webdriver.common.action_chains import ActionChains
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions
+from selenium.webdriver.support.ui import WebDriverWait
+from seleniumrequests.request import RequestsSessionMixin
+
+cfg = load_config()
+
+
+def set_server_url(server_url):
+    """Set web driver server URL using value loaded from test config file."""
+    MyCustomWebDriver.server_url = server_url
+
+
+class MyCustomWebDriver(RequestsSessionMixin, webdriver.Firefox):
+    @property
+    def server_url(self):
+        if not hasattr(self, "_server_url"):
+            raise NotImplementedError(
+                "Please first set the web driver URL using `set_server_url`"
+            )
+        return self._server_url
+
+    @server_url.setter
+    def server_url(self, value):
+        self._server_url = value
+
+    def get(self, uri):
+        webdriver.Firefox.get(self, self.server_url + uri)
+        try:
+            self.find_element(By.ID, "websocketStatus")
+            self.wait_for_xpath(
+                "//*[@id='websocketStatus' and contains(@title,'connected')]"
+            )
+        except NoSuchElementException:
+            pass
+
+    def wait_for_xpath(self, xpath, timeout=10):
+        return WebDriverWait(self, timeout).until(
+            expected_conditions.presence_of_element_located((By.XPATH, xpath))
+        )
+
+    def wait_for_css(self, css, timeout=10):
+        return WebDriverWait(self, timeout).until(
+            expected_conditions.presence_of_element_located((By.CSS_SELECTOR, css))
+        )
+
+    def wait_for_xpath_to_appear(self, xpath, timeout=10):
+        return WebDriverWait(self, timeout).until_not(
+            expected_conditions.invisibility_of_element((By.XPATH, xpath))
+        )
+
+    def wait_for_xpath_to_disappear(self, xpath, timeout=10):
+        return WebDriverWait(self, timeout).until(
+            expected_conditions.invisibility_of_element((By.XPATH, xpath))
+        )
+
+    def wait_for_css_to_disappear(self, css, timeout=10):
+        return WebDriverWait(self, timeout).until(
+            expected_conditions.invisibility_of_element((By.CSS_SELECTOR, css))
+        )
+
+    def wait_for_xpath_to_be_clickable(self, xpath, timeout=10):
+        return WebDriverWait(self, timeout).until(
+            expected_conditions.element_to_be_clickable((By.XPATH, xpath))
+        )
+
+    def wait_for_xpath_to_be_unclickable(self, xpath, timeout=10):
+        return WebDriverWait(self, timeout).until_not(
+            expected_conditions.element_to_be_clickable((By.XPATH, xpath))
+        )
+
+    def wait_for_css_to_be_clickable(self, css, timeout=10):
+        return WebDriverWait(self, timeout).until(
+            expected_conditions.element_to_be_clickable((By.CSS_SELECTOR, css))
+        )
+
+    def wait_for_css_to_be_unclickable(self, css, timeout=10):
+        return WebDriverWait(self, timeout).until_not(
+            expected_conditions.element_to_be_clickable((By.CSS_SELECTOR, css))
+        )
+
+    def scroll_to_element(self, element, scroll_parent=False):
+        scroll_script = (
+            """
+                arguments[0].scrollIntoView();
+            """
+            if scroll_parent
+            else """
+            const viewPortHeight = Math.max(document.documentElement.clientHeight, window.innerHeight || 0);
+            const elementTop = arguments[0].getBoundingClientRect().top;
+            window.scrollBy(0, elementTop - (viewPortHeight / 2));
+            """
+        )
+        self.execute_script(scroll_script, element)
+
+    def scroll_to_element_and_click(self, element, timeout=10, scroll_parent=False):
+        self.scroll_to_element(element, scroll_parent=scroll_parent)
+        ActionChains(self).move_to_element(element).perform()
+
+        try:
+            return element.click()
+        except ElementClickInterceptedException:
+            pass
+        except StaleElementReferenceException:
+            pass
+
+        try:
+            return self.execute_script("arguments[0].click();", element)
+        except JavascriptException:
+            pass
+        except StaleElementReferenceException:
+            pass
+
+        # Tried to click something that's not a button, try sending
+        # a mouse click to that coordinate
+        ActionChains(self).click().perform()
+
+    def click_xpath(self, xpath, wait_clickable=True, timeout=10, scroll_parent=False):
+        if wait_clickable:
+            element = self.wait_for_xpath_to_be_clickable(xpath, timeout=timeout)
+        else:
+            element = self.wait_for_xpath(xpath)
+        return self.scroll_to_element_and_click(element, scroll_parent=scroll_parent)
+
+    def click_css(self, css, timeout=10, scroll_parent=False):
+        element = self.wait_for_css_to_be_clickable(css, timeout=timeout)
+        return self.scroll_to_element_and_click(element, scroll_parent=scroll_parent)
+
+
+@pytest.fixture(scope="session")
+def driver(request):
+    import shutil
+
+    from selenium import webdriver
+    from webdriver_manager.firefox import GeckoDriverManager
+
+    options = webdriver.FirefoxOptions()
+    if str(os.getenv("FRONTEND_TEST_HEADLESS", "0")).strip().lower() in (
+        "1",
+        "true",
+        "t",
+        "yes",
+        "y",
+    ):
+        options.add_argument("-headless")
+    options.set_preference("devtools.console.stdout.content", True)
+    options.set_preference("browser.download.manager.showWhenStarting", False)
+    options.set_preference("browser.download.folderList", 2)
+    options.set_preference(
+        "browser.download.dir", os.path.abspath(cfg["paths.downloads_folder"])
+    )
+    options.set_preference(
+        "browser.helperApps.neverAsk.saveToDisk",
+        (
+            "text/csv,text/plain,application/octet-stream,"
+            "text/comma-separated-values,text/html"
+        ),
+    )
+
+    executable_path = shutil.which("geckodriver")
+    if executable_path is None:
+        executable_path = GeckoDriverManager().install()
+    service = webdriver.firefox.service.Service(executable_path=executable_path)
+
+    driver = MyCustomWebDriver(options=options, service=service)
+    driver.set_window_size(1920, 1200)
+    login(driver)
+
+    yield driver
+
+    driver.close()
+
+
+def login(driver):
+    username_xpath = '//*[contains(string(),"testuser-cesium-ml-org")]'
+
+    driver.get("/")
+    try:
+        driver.wait_for_xpath(username_xpath, 0.25)
+        return  # Already logged in
+    except TimeoutException:
+        pass
+
+    try:
+        element = driver.wait_for_xpath('//a[contains(@href,"/login/iam-oauth2")]', 20)
+        element.click()
+    except TimeoutException:
+        pass
+
+    try:
+        driver.wait_for_xpath(username_xpath, 5)
+    except TimeoutException:
+        raise TimeoutException("Login failed:\n" + driver.page_source)
+
+
+@pytest.fixture(scope="function", autouse=True)
+def reset_state(request):
+    def teardown():
+        models.DBSession().rollback()
+
+    request.addfinalizer(teardown)