From f787434c6af02c201061760cf66b8e8493a38c85 Mon Sep 17 00:00:00 2001 From: Yajing Tang Date: Mon, 23 Apr 2018 15:11:56 -0500 Subject: [PATCH] feat(acl): add acl table (#109) * feat(acl): add acl table * fix(acl): add data migration * fix(java): try another package --- .travis.yml | 2 +- indexd/index/blueprint.py | 10 ++++ indexd/index/drivers/alchemy.py | 100 ++++++++++++++++++++++++++++---- indexd/index/schema.py | 12 ++++ openapis/swagger.yaml | 20 +++++++ tests/test_client.py | 22 ++++++- tests/test_schema_migration.py | 23 +++++++- 7 files changed, 175 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 150f5cc1..33f73027 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ python: addons: apt: packages: - - oracle-java8-installer + - oracle-java8-set-default sudo: false diff --git a/indexd/index/blueprint.py b/indexd/index/blueprint.py index aac6aeeb..708a6485 100644 --- a/indexd/index/blueprint.py +++ b/indexd/index/blueprint.py @@ -69,6 +69,8 @@ def get_index(): urls = flask.request.args.getlist('url') + acl = flask.request.args.getlist('url') + file_name = flask.request.args.get('file_name') version = flask.request.args.get('version') @@ -92,6 +94,7 @@ def get_index(): file_name=file_name, version=version, urls=urls, + acl=acl, hashes=hashes, metadata=metadata, ) @@ -104,6 +107,7 @@ def get_index(): 'file_name': file_name, 'version': version, 'urls': urls, + 'acl': acl, 'hashes': hashes, 'metadata': metadata, } @@ -188,6 +192,7 @@ def post_index_record(): form = flask.request.json['form'] size = flask.request.json['size'] urls = flask.request.json['urls'] + acl = flask.request.json.get('acl', []) hashes = flask.request.json['hashes'] file_name = flask.request.json.get('file_name') @@ -203,6 +208,7 @@ def post_index_record(): metadata=metadata, version=version, urls=urls, + acl=acl, hashes=hashes, baseid=baseid, ) @@ -231,6 +237,7 @@ def put_index_record(record): file_name = flask.request.json.get('file_name') version = flask.request.json.get('version') urls = flask.request.json.get('urls') + acl = flask.request.json.get('acl') metadata = flask.request.json.get('metadata') did, baseid, rev = blueprint.index_driver.update( @@ -239,6 +246,7 @@ def put_index_record(record): file_name=file_name, version=version, urls=urls, + acl=acl, metadata=metadata, ) @@ -281,6 +289,7 @@ def add_index_record_version(record): form = flask.request.json['form'] size = flask.request.json['size'] urls = flask.request.json['urls'] + acl = flask.request.json.get('acl', []) hashes = flask.request.json['hashes'] file_name = flask.request.json.get('file_name', None) metadata = flask.request.json.get('metadata', None) @@ -292,6 +301,7 @@ def add_index_record_version(record): new_did=new_did, size=size, urls=urls, + acl=acl, file_name=file_name, metadata=metadata, version=version, diff --git a/indexd/index/drivers/alchemy.py b/indexd/index/drivers/alchemy.py index cc3ac365..1b968add 100644 --- a/indexd/index/drivers/alchemy.py +++ b/indexd/index/drivers/alchemy.py @@ -64,6 +64,12 @@ class IndexRecord(Base): cascade='all, delete-orphan', ) + acl = relationship( + 'IndexRecordACE', + backref='index_record', + cascade='all, delete-orphan', + ) + hashes = relationship( 'IndexRecordHash', backref='index_record', @@ -92,7 +98,25 @@ class IndexRecordUrl(Base): backref='index_record_url', cascade='all, delete-orphan', ) - Index('index_record_url_idx', 'did') + __table_args__ = ( + Index('index_record_url_idx', 'did'), + ) + + +class IndexRecordACE(Base): + ''' + index record access control entry representation. + ''' + + __tablename__ = 'index_record_ace' + + did = Column(String, ForeignKey('index_record.did'), primary_key=True) + # access control entry + ace = Column(String, primary_key=True) + + __table_args__ = ( + Index('index_record_ace_idx', 'did'), + ) class IndexRecordMetadata(Base): @@ -104,9 +128,9 @@ class IndexRecordMetadata(Base): key = Column(String, primary_key=True) did = Column(String, ForeignKey('index_record.did'), primary_key=True) value = Column(String) - Index('index_record_metadata_idx', 'did') - Index('__did_key_idx', 'did', 'key') - + __table_args__ = ( + Index('index_record_metadata_idx', 'did'), + ) class IndexRecordUrlMetadata(Base): """ @@ -121,9 +145,9 @@ class IndexRecordUrlMetadata(Base): __table_args__ = ( ForeignKeyConstraint(['did', 'url'], ['index_record_url.did', 'index_record_url.url']), + Index('index_record_url_metadata_idx', 'did'), ) - Index('index_record_url_metadata_idx', 'did') - Index('__did_url_key_idx', 'did', 'url', 'key') + class IndexRecordHash(Base): @@ -135,7 +159,9 @@ class IndexRecordHash(Base): did = Column(String, ForeignKey('index_record.did'), primary_key=True) hash_type = Column(String, primary_key=True) hash_value = Column(String) - Index('index_record_hash_idx', 'did') + __table_args__ = ( + Index('index_record_hash_idx', 'did'), + ) class SQLAlchemyIndexDriver(IndexDriverABC): @@ -196,6 +222,7 @@ def ids(self, start=None, size=None, urls=None, + acl=None, hashes=None, file_name=None, version=None, @@ -223,6 +250,11 @@ def ids(self, for u in urls: query = query.filter(IndexRecordUrl.url == u) + if acl is not None and acl: + query = query.join(IndexRecord.acl) + for u in acl: + query = query.filter(IndexRecordACE.ace == u) + if hashes is not None and hashes: for h, v in hashes.items(): sub = session.query(IndexRecordHash.did) @@ -285,15 +317,18 @@ def add(self, metadata=None, version=None, urls=None, + acl=None, hashes=None, baseid=None): """ - Creates a new record given size, urls, hashes, metadata, file name and version + Creates a new record given size, urls, acl, hashes, metadata, file name and version if did is provided, update the new record with the did otherwise create it """ if urls is None: urls = [] + if acl is None: + acl = [] if hashes is None: hashes = {} if metadata is None: @@ -323,6 +358,11 @@ def add(self, url=url, ) for url in urls] + record.acl = [IndexRecordACE( + did=record.did, + ace=ace, + ) for ace in acl] + record.hashes = [IndexRecordHash( did=record.did, hash_type=h, @@ -374,6 +414,7 @@ def get(self, did): version = record.version urls = [u.url for u in record.urls] + acl = [u.ace for u in record.acl] hashes = {h.hash_type: h.hash_value for h in record.hashes} metadata = {m.key: m.value for m in record.index_metadata} @@ -388,6 +429,7 @@ def get(self, did): 'file_name': file_name, 'version': version, 'urls': urls, + 'acl': acl, 'hashes': hashes, 'metadata': metadata, 'form': form, @@ -398,7 +440,7 @@ def get(self, did): return ret def update(self, - did, rev, urls=None, file_name=None, + did, rev, urls=None, acl=None, file_name=None, version=None, metadata=None): """ Updates an existing record with new values. @@ -426,6 +468,15 @@ def update(self, url=url ) for url in urls] + if acl is not None: + for ace in record.acl: + session.delete(ace) + + record.acl = [IndexRecordACE( + did=record.did, + ace=ace + ) for ace in acl] + if metadata is not None: for md_record in record.index_metadata: session.delete(md_record) @@ -477,12 +528,15 @@ def add_version(self, metadata=None, version=None, urls=None, + acl=None, hashes=None): """ Add a record version given did """ if urls is None: urls = [] + if acl is None: + acl = [] if hashes is None: hashes = {} if metadata is None: @@ -514,6 +568,11 @@ def add_version(self, url=url, ) for url in urls] + record.acl = [IndexRecordACE( + did=record.did, + ace=ace, + ) for ace in acl] + record.hashes = [IndexRecordHash( did=record.did, hash_type=h, @@ -568,6 +627,7 @@ def get_all_versions(self, did): file_name = record.file_name version = record.version urls = [u.url for u in record.urls] + acl = [u.ace for u in record.acl] hashes = {h.hash_type: h.hash_value for h in record.hashes} metadata = {m.key: m.value for m in record.index_metadata} @@ -583,6 +643,7 @@ def get_all_versions(self, did): 'metadata': metadata, 'version': version, 'urls': urls, + 'acl': acl, 'hashes': hashes, 'form': form, 'created_date': created_date, @@ -628,6 +689,7 @@ def get_latest_version(self, did): version = record.version urls = [u.url for u in record.urls] + acl = [u.ace for u in record.acl] hashes = {h.hash_type: h.hash_value for h in record.hashes} created_date = record.created_date.isoformat() @@ -642,6 +704,7 @@ def get_latest_version(self, did): 'metadata': metadata, 'version': version, 'urls': urls, + 'acl': acl, 'hashes': hashes, 'form': form, 'created_date': created_date, @@ -792,7 +855,24 @@ def migrate_5(session, **kwargs): "CREATE INDEX {tb}_idx ON {tb} ( did )" .format(tb=IndexRecordUrlMetadata.__tablename__)) + +def migrate_6(session, **kwargs): + existing_acls = ( + session.query(IndexRecordMetadata).filter_by(key='acl').yield_per(1000) + ) + for metadata in existing_acls: + acl = metadata.value.split(',') + for ace in acl: + entry = IndexRecordACE( + did=metadata.did, + ace=ace) + session.add(entry) + session.delete(metadata) + + # ordered schema migration functions that the index should correspond to # CURRENT_SCHEMA_VERSION - 1 when it's written -SCHEMA_MIGRATION_FUNCTIONS = [migrate_1, migrate_2, migrate_3, migrate_4, migrate_5] +SCHEMA_MIGRATION_FUNCTIONS = [ + migrate_1, migrate_2, migrate_3, migrate_4, migrate_5, + migrate_6] CURRENT_SCHEMA_VERSION = len(SCHEMA_MIGRATION_FUNCTIONS) diff --git a/indexd/index/schema.py b/indexd/index/schema.py index 91123c48..d53c5f9f 100644 --- a/indexd/index/schema.py +++ b/indexd/index/schema.py @@ -44,6 +44,12 @@ "type": "string" } }, + "acl": { + "type": "array", + "items": { + "type": "string" + } + }, "did": { "type": "string", "pattern": "^.*[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$" @@ -124,6 +130,12 @@ "type": "string" } }, + "acl": { + "type": "array", + "items": { + "type": "string" + } + }, "file_name": { "type": "string" }, diff --git a/openapis/swagger.yaml b/openapis/swagger.yaml index 8c4669d4..2f342a27 100644 --- a/openapis/swagger.yaml +++ b/openapis/swagger.yaml @@ -130,6 +130,10 @@ paths: type: array items: type: string + acl: + type: array + items: + type: string hashes: $ref: '#/definitions/HashInfo' security: [] @@ -552,6 +556,10 @@ definitions: type: array items: type: string + acl: + type: array + items: + type: string hashes: $ref: '#/definitions/HashInfo' InputInfo: @@ -587,6 +595,10 @@ definitions: type: array items: type: string + acl: + type: array + items: + type: string hashes: $ref: '#/definitions/HashInfo' UpdateInputInfo: @@ -606,6 +618,10 @@ definitions: type: array items: type: string + acl: + type: array + items: + type: string OutputRef: type: object properties: @@ -718,6 +734,10 @@ definitions: type: array items: type: string + acl: + type: array + items: + type: string hashes: $ref: '#/definitions/HashInfo' updated_date_by: diff --git a/tests/test_client.py b/tests/test_client.py index 19cfcdc1..395e7c13 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -56,6 +56,9 @@ def test_index_create(swg_index_client): result = swg_index_client.add_entry(data) assert result.did assert result.baseid == data['baseid'] + r = swg_index_client.get_entry(result.did) + assert r.acl == [] + def test_index_get(swg_index_client): @@ -126,6 +129,19 @@ def test_index_create_with_valid_did(swg_index_client): assert result.did == '3d313755-cbb4-4b08-899d-7bbac1f6e67d' +def test_index_create_with_acl(swg_index_client): + data = { + 'acl': ['a', 'b'], + 'form': 'object', + 'size': 123, + 'urls': ['s3://endpointurl/bucket/key'], + 'hashes': {'md5': '8b9942cf415384b27cadf1f4d2d682e5'}} + + r = swg_index_client.add_entry(data) + result = swg_index_client.get_entry(r.did) + assert result.acl == ['a', 'b'] + + def test_index_create_with_invalid_did(swg_index_client): data = get_doc() @@ -199,9 +215,12 @@ def test_index_update(swg_index_client): del dataNew['form'] dataNew['metadata'] = {'test': 'abcd'} dataNew['version'] = 'ver123' + dataNew['acl'] = ['a', 'b'] r2 = swg_index_client.update_entry(r.did, rev=r.rev, body=dataNew) assert r2.rev != r.rev - assert swg_index_client.get_entry(r.did).metadata == dataNew['metadata'] + result = swg_index_client.get_entry(r.did) + assert result.metadata == dataNew['metadata'] + assert result.acl == dataNew['acl'] data = get_doc() data['did'] = 'cdis:3d313755-cbb4-4b08-899d-7bbac1f6e67d' @@ -248,6 +267,7 @@ def test_create_index_version(swg_index_client): 'size': 244, 'urls': ['s3://endpointurl/bucket2/key'], 'hashes': {'md5': '8b9942cf415384b27cadf1f4d2d981f5'}, + 'acl': ['a'], } r2 = swg_index_client.add_new_version(r.did, body=dataNew) diff --git a/tests/test_schema_migration.py b/tests/test_schema_migration.py index 6c1261b1..b5042ad0 100644 --- a/tests/test_schema_migration.py +++ b/tests/test_schema_migration.py @@ -5,7 +5,7 @@ import sqlite3 import tests.util as util from indexd.index.drivers.alchemy import ( - SQLAlchemyIndexDriver, IndexSchemaVersion) + SQLAlchemyIndexDriver, IndexSchemaVersion, migrate_6) from indexd.alias.drivers.alchemy import ( SQLAlchemyAliasDriver, AliasSchemaVersion) @@ -56,6 +56,24 @@ def update_version_table_for_testing(db, tb_name, val): conn.commit() +def test_migrate_6(swg_index_client, indexd_server): + data = { + 'form': 'object', + 'size': 123, + 'urls': ['s3://endpointurl/bucket/key'], + 'metadata': { + 'acl': 'a,b' + }, + 'hashes': {'md5': '8b9942cf415384b27cadf1f4d2d682e5'} + } + + r = swg_index_client.add_entry(data) + with settings['config']['INDEX']['driver'].session as session: + migrate_6(session) + r = swg_index_client.get_entry(r.did) + assert r.acl == ['a', 'b'] + assert r.metadata == {} + @util.removes('index.sq3') def test_migrate_index(): def test_migrate_index_internal(monkeypatch): @@ -225,4 +243,5 @@ def test_migrate_index_versioning(monkeypatch): def test_schema_version(): - assert CURRENT_SCHEMA_VERSION == len(SCHEMA_MIGRATION_FUNCTIONS) \ No newline at end of file + assert CURRENT_SCHEMA_VERSION == len(SCHEMA_MIGRATION_FUNCTIONS) +