feat(acl): add acl table (#109)

* feat(acl): add acl table

* fix(acl): add data migration

* fix(java): try another package

.travis.yml

@@ -6,7 +6,7 @@ python:
 addons:
   apt:
     packages:
-      - oracle-java8-installer
+      - oracle-java8-set-default
 
 sudo: false
 

indexd/index/blueprint.py

@@ -69,6 +69,8 @@ def get_index():
 
     urls = flask.request.args.getlist('url')
 
+    acl = flask.request.args.getlist('url')
+
     file_name = flask.request.args.get('file_name')
 
     version = flask.request.args.get('version')
@@ -92,6 +94,7 @@ def get_index():
         file_name=file_name,
         version=version,
         urls=urls,
+        acl=acl,
         hashes=hashes,
         metadata=metadata,
     )
@@ -104,6 +107,7 @@ def get_index():
         'file_name': file_name,
         'version': version,
         'urls': urls,
+        'acl': acl,
         'hashes': hashes,
         'metadata': metadata,
     }
@@ -188,6 +192,7 @@ def post_index_record():
     form = flask.request.json['form']
     size = flask.request.json['size']
     urls = flask.request.json['urls']
+    acl = flask.request.json.get('acl', [])
 
     hashes = flask.request.json['hashes']
     file_name = flask.request.json.get('file_name')
@@ -203,6 +208,7 @@ def post_index_record():
         metadata=metadata,
         version=version,
         urls=urls,
+        acl=acl,
         hashes=hashes,
         baseid=baseid,
     )
@@ -231,6 +237,7 @@ def put_index_record(record):
     file_name = flask.request.json.get('file_name')
     version = flask.request.json.get('version')
     urls = flask.request.json.get('urls')
+    acl = flask.request.json.get('acl')
     metadata = flask.request.json.get('metadata')
 
     did, baseid, rev = blueprint.index_driver.update(
@@ -239,6 +246,7 @@ def put_index_record(record):
         file_name=file_name,
         version=version,
         urls=urls,
+        acl=acl,
         metadata=metadata,
     )
 
@@ -281,6 +289,7 @@ def add_index_record_version(record):
     form = flask.request.json['form']
     size = flask.request.json['size']
     urls = flask.request.json['urls']
+    acl = flask.request.json.get('acl', [])
     hashes = flask.request.json['hashes']
     file_name = flask.request.json.get('file_name', None)
     metadata = flask.request.json.get('metadata', None)
@@ -292,6 +301,7 @@ def add_index_record_version(record):
         new_did=new_did,
         size=size,
         urls=urls,
+        acl=acl,
         file_name=file_name,
         metadata=metadata,
         version=version,

indexd/index/drivers/alchemy.py

@@ -64,6 +64,12 @@ class IndexRecord(Base):
         cascade='all, delete-orphan',
     )
 
+    acl = relationship(
+        'IndexRecordACE',
+        backref='index_record',
+        cascade='all, delete-orphan',
+    )
+
     hashes = relationship(
         'IndexRecordHash',
         backref='index_record',
@@ -92,7 +98,25 @@ class IndexRecordUrl(Base):
         backref='index_record_url',
         cascade='all, delete-orphan',
     )
-    Index('index_record_url_idx', 'did')
+    __table_args__ = (
+        Index('index_record_url_idx', 'did'),
+    )
+
+
+class IndexRecordACE(Base):
+    '''
+    index record access control entry representation.
+    '''
+
+    __tablename__ = 'index_record_ace'
+
+    did = Column(String, ForeignKey('index_record.did'), primary_key=True)
+    # access control entry
+    ace = Column(String, primary_key=True)
+
+    __table_args__ = (
+        Index('index_record_ace_idx', 'did'),
+    )
 
 
 class IndexRecordMetadata(Base):
@@ -104,9 +128,9 @@ class IndexRecordMetadata(Base):
     key = Column(String, primary_key=True)
     did = Column(String, ForeignKey('index_record.did'), primary_key=True)
     value = Column(String)
-    Index('index_record_metadata_idx', 'did')
-    Index('__did_key_idx', 'did', 'key')
-
+    __table_args__ = (
+        Index('index_record_metadata_idx', 'did'),
+    )
 
 class IndexRecordUrlMetadata(Base):
     """
@@ -121,9 +145,9 @@ class IndexRecordUrlMetadata(Base):
     __table_args__ = (
         ForeignKeyConstraint(['did', 'url'],
                              ['index_record_url.did', 'index_record_url.url']),
+        Index('index_record_url_metadata_idx', 'did'),
     )
-    Index('index_record_url_metadata_idx', 'did')
-    Index('__did_url_key_idx', 'did', 'url', 'key')
+
 
 
 class IndexRecordHash(Base):
@@ -135,7 +159,9 @@ class IndexRecordHash(Base):
     did = Column(String, ForeignKey('index_record.did'), primary_key=True)
     hash_type = Column(String, primary_key=True)
     hash_value = Column(String)
-    Index('index_record_hash_idx', 'did')
+    __table_args__ = (
+        Index('index_record_hash_idx', 'did'),
+    )
 
 
 class SQLAlchemyIndexDriver(IndexDriverABC):
@@ -196,6 +222,7 @@ def ids(self,
             start=None,
             size=None,
             urls=None,
+            acl=None,
             hashes=None,
             file_name=None,
             version=None,
@@ -223,6 +250,11 @@ def ids(self,
                 for u in urls:
                     query = query.filter(IndexRecordUrl.url == u)
 
+            if acl is not None and acl:
+                query = query.join(IndexRecord.acl)
+                for u in acl:
+                    query = query.filter(IndexRecordACE.ace == u)
+
             if hashes is not None and hashes:
                 for h, v in hashes.items():
                     sub = session.query(IndexRecordHash.did)
@@ -285,15 +317,18 @@ def add(self,
             metadata=None,
             version=None,
             urls=None,
+            acl=None,
             hashes=None,
             baseid=None):
         """
-        Creates a new record given size, urls, hashes, metadata, file name and version
+        Creates a new record given size, urls, acl, hashes, metadata, file name and version
         if did is provided, update the new record with the did otherwise create it
         """
 
         if urls is None:
             urls = []
+        if acl is None:
+            acl = []
         if hashes is None:
             hashes = {}
         if metadata is None:
@@ -323,6 +358,11 @@ def add(self,
                 url=url,
             ) for url in urls]
 
+            record.acl = [IndexRecordACE(
+                did=record.did,
+                ace=ace,
+            ) for ace in acl]
+
             record.hashes = [IndexRecordHash(
                 did=record.did,
                 hash_type=h,
@@ -374,6 +414,7 @@ def get(self, did):
             version = record.version
 
             urls = [u.url for u in record.urls]
+            acl = [u.ace for u in record.acl]
             hashes = {h.hash_type: h.hash_value for h in record.hashes}
             metadata = {m.key: m.value for m in record.index_metadata}
 
@@ -388,6 +429,7 @@ def get(self, did):
                 'file_name': file_name,
                 'version': version,
                 'urls': urls,
+                'acl': acl,
                 'hashes': hashes,
                 'metadata': metadata,
                 'form': form,
@@ -398,7 +440,7 @@ def get(self, did):
         return ret
 
     def update(self,
-               did, rev, urls=None, file_name=None,
+               did, rev, urls=None, acl=None, file_name=None,
                version=None, metadata=None):
         """
         Updates an existing record with new values.
@@ -426,6 +468,15 @@ def update(self,
                     url=url
                 ) for url in urls]
 
+            if acl is not None:
+                for ace in record.acl:
+                    session.delete(ace)
+
+                record.acl = [IndexRecordACE(
+                    did=record.did,
+                    ace=ace
+                ) for ace in acl]
+
             if metadata is not None:
                 for md_record in record.index_metadata:
                     session.delete(md_record)
@@ -477,12 +528,15 @@ def add_version(self,
                     metadata=None,
                     version=None,
                     urls=None,
+                    acl=None,
                     hashes=None):
         """
         Add a record version given did
         """
         if urls is None:
             urls = []
+        if acl is None:
+            acl = []
         if hashes is None:
             hashes = {}
         if metadata is None:
@@ -514,6 +568,11 @@ def add_version(self,
                 url=url,
             ) for url in urls]
 
+            record.acl = [IndexRecordACE(
+                did=record.did,
+                ace=ace,
+            ) for ace in acl]
+
             record.hashes = [IndexRecordHash(
                 did=record.did,
                 hash_type=h,
@@ -568,6 +627,7 @@ def get_all_versions(self, did):
                 file_name = record.file_name
                 version = record.version
                 urls = [u.url for u in record.urls]
+                acl = [u.ace for u in record.acl]
                 hashes = {h.hash_type: h.hash_value for h in record.hashes}
                 metadata = {m.key: m.value for m in record.index_metadata}
 
@@ -583,6 +643,7 @@ def get_all_versions(self, did):
                     'metadata': metadata,
                     'version': version,
                     'urls': urls,
+                    'acl': acl,
                     'hashes': hashes,
                     'form': form,
                     'created_date': created_date,
@@ -628,6 +689,7 @@ def get_latest_version(self, did):
             version = record.version
 
             urls = [u.url for u in record.urls]
+            acl = [u.ace for u in record.acl]
             hashes = {h.hash_type: h.hash_value for h in record.hashes}
 
             created_date = record.created_date.isoformat()
@@ -642,6 +704,7 @@ def get_latest_version(self, did):
                 'metadata': metadata,
                 'version': version,
                 'urls': urls,
+                'acl': acl,
                 'hashes': hashes,
                 'form': form,
                 'created_date': created_date,
@@ -792,7 +855,24 @@ def migrate_5(session, **kwargs):
         "CREATE INDEX {tb}_idx ON {tb} ( did )"
             .format(tb=IndexRecordUrlMetadata.__tablename__))
 
+
+def migrate_6(session, **kwargs):
+    existing_acls = (
+        session.query(IndexRecordMetadata).filter_by(key='acl').yield_per(1000)
+    )
+    for metadata in existing_acls:
+        acl = metadata.value.split(',')
+        for ace in acl:
+            entry = IndexRecordACE(
+                did=metadata.did,
+                ace=ace)
+            session.add(entry)
+            session.delete(metadata)
+
+
 # ordered schema migration functions that the index should correspond to
 # CURRENT_SCHEMA_VERSION - 1 when it's written
-SCHEMA_MIGRATION_FUNCTIONS = [migrate_1, migrate_2, migrate_3, migrate_4, migrate_5]
+SCHEMA_MIGRATION_FUNCTIONS = [
+    migrate_1, migrate_2, migrate_3, migrate_4, migrate_5,
+    migrate_6]
 CURRENT_SCHEMA_VERSION = len(SCHEMA_MIGRATION_FUNCTIONS)

indexd/index/schema.py

@@ -44,6 +44,12 @@
         "type": "string"
       }
     },
+    "acl": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
     "did": {
       "type": "string",
       "pattern": "^.*[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$"
@@ -124,6 +130,12 @@
         "type": "string"
       }
     },
+    "acl": {
+      "type": "array",
+      "items": {
+        "type": "string"
+      }
+    },
     "file_name": {
         "type": "string"
     },