fix(migrate): improve acl->authz migration, remove deprecated endpoin…

…ts (#336)

* fix(migrate_acl): allow loading and preference of authz mapping from fence configuration

* fix(migrate): correct function call, make pyyaml main dep, update deps

* chore(migrate): ensure logging is on for migrate script

* fix(migrate): use user.yaml, not fence config

* fix(fence): finally remove deprecated fence client and access endpoints

* chore(docs): update to remove deprecated and removed endpoint

Co-authored-by: Alexander VT <[email protected]>

bin/migrate_acl_authz.py

@@ -34,8 +34,9 @@
 
 from indexd.index.drivers.alchemy import IndexRecord, IndexRecordAuthz
 
+from yaml import safe_load
 
-logger = get_logger("migrate_acl_authz")
+logger = get_logger("migrate_acl_authz", log_level="debug")
 
 
 def main():
@@ -49,7 +50,10 @@ def main():
     driver = settings["config"]["INDEX"]["driver"]
     try:
         acl_converter = ACLConverter(
-            args.arborist, getattr(args, "sheepdog"), getattr(args, "use_tags")
+            args.arborist,
+            getattr(args, "sheepdog"),
+            getattr(args, "use_tags"),
+            getattr(args, "user_yaml_path"),
         )
     except EnvironmentError:
         logger.error("can't continue without database connection")
@@ -119,11 +123,18 @@ def parse_args():
         dest="start_did",
         help="did to start at (records processed in lexographical order)",
     )
+    parser.add_argument(
+        "--user-yaml-path",
+        dest="user_yaml_path",
+        help="path to user yaml for pulling authz mapping",
+    )
     return parser.parse_args()
 
 
 class ACLConverter(object):
-    def __init__(self, arborist_url, sheepdog_db=None, use_tags=False):
+    def __init__(
+        self, arborist_url, sheepdog_db=None, use_tags=False, user_yaml_path=None
+    ):
         self.arborist_url = arborist_url.rstrip("/")
         self.programs = set()
         self.projects = dict()
@@ -134,6 +145,21 @@ def __init__(self, arborist_url, sheepdog_db=None, use_tags=False):
         else:
             logger.info("not using any auth namespace")
         self.use_sheepdog_db = bool(sheepdog_db)
+        self.mapping = {}
+
+        if user_yaml_path:
+            with open(user_yaml_path, "r") as f:
+                user_yaml = safe_load(f)
+                user_yaml_authz = user_yaml.get("authz", dict())
+                if not user_yaml_authz:
+                    user_yaml_authz = user_yaml.get("rbac", dict())
+
+                project_to_resource = user_yaml_authz.get(
+                    "user_project_to_resource", dict()
+                )
+                self.mapping = project_to_resource
+
+        logger.info(f"got mapping: {self.mapping}")
 
         # if "use_tags" is True, map resource paths to tags in arborist so
         # we can save http calls
@@ -194,6 +220,10 @@ def acl_to_authz(self, record):
             if not acl_item:
                 # ignore empty string
                 continue
+            # prefer user.yaml authz mapping if provided
+            elif acl_item in self.mapping:
+                path = self.mapping[acl_item]
+                projects_found += 1
             elif acl_item == "*":
                 # if there's a * it should just be open. return early
                 path = "/open"

indexd/app.py

@@ -7,7 +7,6 @@
 from .guid.blueprint import blueprint as indexd_drs_blueprint
 from .blueprint import blueprint as cross_blueprint
 
-from indexd.fence_client import FenceClient
 from indexd.urls.blueprint import blueprint as index_urls_blueprint
 
 import os
@@ -22,10 +21,6 @@ def app_init(app, settings=None):
         from .default_settings import settings
     app.config.update(settings["config"])
     app.auth = settings["auth"]
-    app.fence_client = FenceClient(
-        url=os.environ.get("PRESIGNED_FENCE_URL")
-        or "http://presigned-url-fence-service"
-    )
     app.hostname = os.environ.get("HOSTNAME") or "http://example.io"
     app.register_blueprint(indexd_bulk_blueprint)
     app.register_blueprint(indexd_index_blueprint)

indexd/drs/blueprint.py

@@ -64,26 +64,6 @@ def list_drs_records():
     return flask.jsonify(ret), 200
 
 
-@blueprint.route(
-    "/ga4gh/drs/v1/objects/<path:object_id>/access",
-    defaults={"access_id": None},
-    methods=["GET"],
-)
-@blueprint.route(
-    "/ga4gh/drs/v1/objects/<path:object_id>/access/<path:access_id>", methods=["GET"]
-)
-def get_signed_url(object_id, access_id):
-    if not access_id:
-        raise (UserError("Access ID/Protocol is required."))
-    res = flask.current_app.fence_client.get_signed_url_for_object(
-        object_id=object_id, access_id=access_id
-    )
-    if not res:
-        raise IndexNoRecordFound("No signed url found")
-
-    return res, 200
-
-
 def create_drs_uri(did):
     """
     Return ga4gh-compilant drs format uri

openapis/swagger.yaml

@@ -1101,70 +1101,6 @@ paths:
         - authToken: []
       tags:
         - DRS
-  '/ga4gh/drs/v1/objects/{object_id}/access/{access_id}':
-    get:
-      summary: Get a URL for fetching bytes.
-      description: >-
-        Returns a URL that can be used to fetch the bytes of a DrsObject.
-        This method only needs to be called when using an AccessMethod that contains an access_id
-        (e.g., for servers that use signed URLs for fetching object bytes).
-      operationId: GetAccessURL
-      responses:
-        '200':
-          description: The access URL was found successfully.
-          schema:
-            $ref: '#/definitions/AccessURL'
-        # '202':
-        #   description: >
-        #     The operation is delayed and will continue asynchronously.
-        #     The client should retry this same request after the delay specified by Retry-After header.
-        #   headers:
-        #     Retry-After:
-        #       description: >
-        #         Delay in seconds. The client should retry this same request after waiting for this duration.
-        #         To simplify client response processing, this must be an integral relative time in seconds.
-        #         This value SHOULD represent the minimum duration the client should wait before attempting
-        #         the operation again with a reasonable expectation of success. When it is not feasible
-        #         for the server to determine the actual expected delay, the server may return a
-        #         brief, fixed value instead.
-        #       type: integer
-        #       format: int64
-        '400':
-          description: The request is malformed.
-          schema:
-            $ref: '#/definitions/Error'
-        '401':
-          description: The request is unauthorized.
-          schema:
-            $ref: '#/definitions/Error'
-        '404':
-          description: The requested access URL wasn't found
-          schema:
-            $ref: '#/definitions/Error'
-        '403':
-          description: The requester is not authorized to perform this action.
-          schema:
-            $ref: '#/definitions/Error'
-        '500':
-          description: An unexpected error occurred.
-          schema:
-            $ref: '#/definitions/Error'
-      parameters:
-        - name: object_id
-          in: path
-          required: true
-          type: string
-          description: An id of a DrsObject
-        - name: access_id
-          in: path
-          required: true
-          type: string
-          description: An access_id from the access_methods list of a DrsObject
-      security:
-        - authToken: []
-      tags:
-        - DRS
-      x-swagger-router-controller: ga4gh.drs.server
   '/bundle':
     post:
       tags: