feat: Setup transfer collector to fetch from voyager

Author: NickolausDS

xpcs_portal/xpcs_index/api.py

@@ -1,8 +1,10 @@
 import logging
+import globus_sdk
 from django.contrib.auth.decorators import login_required
 from django.http import JsonResponse
 from xpcs_portal.xpcs_index.models import FilenameFilter
 from globus_portal_framework.gclients import load_transfer_client
+from xpcs_portal.xpcs_index.auth import get_globus_app_client
 
 log = logging.getLogger(__name__)
 
@@ -26,7 +28,12 @@ def operation_ls(request, index):
             return JsonResponse({'error': f'Missing param: {item}'}, status=400)
 
     try:
-        tc = load_transfer_client(request.user)
+        globus_app_client = request.GET.get("service_account")
+        if globus_app_client:
+            app = get_globus_app_client(request.user, globus_app_client)
+            tc = globus_sdk.TransferClient(app=app)
+        else:
+            tc = load_transfer_client(request.user)
         response = tc.operation_ls(request.GET["collection"], path=request.GET["path"])
         return JsonResponse(response.data)
     except Exception as e:

xpcs_portal/xpcs_index/auth.py

@@ -0,0 +1,15 @@
+import logging
+import globus_sdk
+from django.conf import settings
+from django.contrib.auth.models import User
+
+log = logging.getLogger(__name__)
+
+
+def get_globus_app_client(user: User, globus_app_client: str):
+    if user.username != "nickolaussaint@globusid.org":
+        raise Exception("Auth not implemented!")
+    log.warning("Danger zone, requesting CC creds on unimplemented auth groups")
+    cc = settings.GLOBUS_APP_FLOWS_AUTHORIZATIONS["confidential_client"][globus_app_client]
+    app = globus_sdk.ClientApp("CC_AUTH", client_id=cc["client_id"], client_secret=cc["client_secret"])
+    return app

xpcs_portal/xpcs_index/collectors.py

@@ -3,10 +3,14 @@
 import urllib
 import copy
 import collections
+import globus_sdk
 from globus_app_flows.collectors.search import SearchCollector
 from globus_app_flows.collectors.transfer import TransferCollector
 from gladier_xpcs.deployments import deployment_map
 from gladier_xpcs.flows.flow_boost import XPCSBoost
+from gladier_xpcs.deployments import BaseDeployment
+from xpcs_portal.xpcs_index.auth import get_globus_app_client
+
 
 log = logging.getLogger(__name__)
 
@@ -22,6 +26,10 @@ def __init__(self, collection=None, path=None, *args, **kwargs):
         # super().__init__(collection="74defd5b-5f61-42fc-bcc4-834c9f376a4f", path="/XPCSDATA/2019-1/comm201901/", user=kwargs["user"])
         super().__init__(collection, path, *args, **kwargs)
 
+    def get_transfer_client(self):
+        app = get_globus_app_client(self.user, "aps8idi-polaris")
+        return globus_sdk.TransferClient(app=app)
+
     def get_files(self, globus_dir: str):
         tc = self.get_transfer_client()
         log.debug(f'Fetching path for {globus_dir}')

xpcs_portal/xpcs_index/templates/xpcs/transfer-selection-form.html

@@ -22,7 +22,7 @@
     function getParent() {
         var cycle = $('#id_cycle').find(":selected").val();
         var cycle_path = path + '/' + cycle
-        $.get({url: "{% url 'xpcs-index:operation-ls' globus_portal_framework.index %}?collection=" + collection + "&path=" + cycle_path,
+        $.get({url: "{% url 'xpcs-index:operation-ls' globus_portal_framework.index %}?collection=" + collection + "&path=" + cycle_path + "&service_account=aps8idi-polaris",
               dataType: "json",
               success: function(data) {updateOptions(data, "#id_parent")}
         });
@@ -36,7 +36,7 @@
           var cycle = $('#id_cycle').find(":selected").val();
           var qmap_path = qmap_basepath + cycle + '/';
           var include_only = $('#id_parent').find(":selected").val();
-          $.get({url: "{% url 'xpcs-index:operation-ls' globus_portal_framework.index %}?collection=" + collection + "&path=" + qmap_path,
+          $.get({url: "{% url 'xpcs-index:operation-ls' globus_portal_framework.index %}?collection=" + collection + "&path=" + qmap_path + "&service_account=aps8idi-polaris",
               dataType: "json",
               success: function(data) {
                 var includes = []
@@ -57,7 +57,7 @@
     $( document ).ready(function() {
         $("#div_id_qmap").hide()
 
-        $.get({url: "{% url 'xpcs-index:operation-ls' globus_portal_framework.index %}?collection=" + collection + "&path=" + path,
+        $.get({url: "{% url 'xpcs-index:operation-ls' globus_portal_framework.index %}?collection=" + collection + "&path=" + path + "&service_account=aps8idi-polaris",
             dataType: "json",
             success: function(data) {data.DATA = data.DATA.reverse(); updateOptions(data, "#id_cycle", exclude=cycle_exclude)},
             error: function(data) {

xpcs_portal/xpcs_index/views.py

@@ -106,8 +106,14 @@ class XPCSComputeTransfer(XPCSReprocessing, BatchCreateView):
     template_name = "xpcs/transfer-selection-form.html"
     form_class = CollectionSelectionForm
     collector = XPCSTransferCollector
-    collection = "74defd5b-5f61-42fc-bcc4-834c9f376a4f"
-    path = "/XPCSDATA/"
+    # Eagle endpoint and path
+    # collection = "74defd5b-5f61-42fc-bcc4-834c9f376a4f"
+    # path = "/XPCSDATA/"
+
+    # Voyager endpoint and path
+    collection = "dc86d51b-81d1-4827-81be-2b5e64ba7dc1"
+    path = "/"
+
     cycle_exclude = [
         "spec_data",
         "partitionMapLibrary",