From aab0e7c1582c63842af5e57003fedfaae40bf9d1 Mon Sep 17 00:00:00 2001 From: Matthew Hilton Date: Tue, 15 Oct 2024 11:48:17 +1000 Subject: [PATCH] feat: add new azure sdk --- README.md | 66 ++-- TESTING.md | 10 +- classes/azure_blob_storage_file_system.php | 33 ++ classes/local/manager.php | 7 +- classes/local/store/azure/stream_wrapper.php | 3 +- .../local/store/azure_blob_storage/client.php | 287 ++++++++++++++++++ .../store/azure_blob_storage/file_system.php | 39 +++ ..._azure_blob_storage_integration_client.php | 60 ++++ classes/tests/test_file_system.php | 7 + tests/object_file_system_test.php | 7 + 10 files changed, 488 insertions(+), 31 deletions(-) create mode 100644 classes/azure_blob_storage_file_system.php create mode 100644 classes/local/store/azure_blob_storage/client.php create mode 100644 classes/local/store/azure_blob_storage/file_system.php create mode 100644 classes/tests/test_azure_blob_storage_integration_client.php diff --git a/README.md b/README.md index 3f52804f..b3fd8f89 100644 --- a/README.md +++ b/README.md @@ -5,31 +5,45 @@ # moodle-tool_objectfs A remote object storage file system for Moodle. Intended to provide a plug-in that can be installed and configured to work with any supported remote object storage solution. -* [Use cases](#use-cases) - * [Offloading large and old files to save money](#offloading-large-and-old-files-to-save-money) - * [Sharing files across moodles to save disk](#sharing-files-across-moodles-to-save-disk) - * [Sharing files across environments to save time](#sharing-files-across-environments-to-save-time) - * [Sharing files with data washed environments](#sharing-files-with-data-washed-environments) -* [Installation](#installation) -* [Compatible object stores](#compatible-object-stores) - * [Amazon S3](#amazon-s3) - * [Minio.io S3](#minio-s3) - * [Google gcs](#google-gcs) - * [Azure Blob Storage](#azure-blob-storage) - * [DigitalOcean Spaces](#digitalocean-spaces) - * [Openstack Object Storage](#openstack-object-storage) -* [Moodle configuration](#moodle-configuration) - * [General Settings](#general-settings) - * [File Transfer settings](#file-transfer-settings) - * [Pre-Signed URLs Settings](#pre-signed-urls-settings) - * [Amazon S3 settings](#amazon-s3-settings) - * [Minio.io S3 settings](#minio-s3-settings) - * [Azure Blob Storage settings](#azure-blob-storage-settings) - * [DigitalOcean Spaces settings](#digitalocean-spaces-settings) -* [Integration testing](#integration-testing) -* [Applying core patches](#applying-core-patches) -* [Crafted by Catalyst IT](#crafted-by-catalyst-it) -* [Contributing and support](#contributing-and-support) +- [moodle-tool\_objectfs](#moodle-tool_objectfs) + - [Use cases](#use-cases) + - [Offloading large and old files to save money](#offloading-large-and-old-files-to-save-money) + - [Sharing files across moodles to save disk](#sharing-files-across-moodles-to-save-disk) + - [Sharing files across environments to save time](#sharing-files-across-environments-to-save-time) + - [Sharing files with data washed environments](#sharing-files-with-data-washed-environments) + - [GDPR](#gdpr) + - [Branches](#branches) + - [Installation](#installation) + - [Compatible object stores](#compatible-object-stores) + - [Amazon S3](#amazon-s3) + - [Minio S3](#minio-s3) + - [Google GCS](#google-gcs) + - [Azure Blob Storage](#azure-blob-storage) + - [DigitalOcean Spaces](#digitalocean-spaces) + - [Openstack Object Storage](#openstack-object-storage) + - [Moodle configuration](#moodle-configuration) + - [General Settings](#general-settings) + - [File Transfer settings](#file-transfer-settings) + - [File System settings](#file-system-settings) + - [Pre-Signed URLs Settings](#pre-signed-urls-settings) + - [Amazon S3 settings](#amazon-s3-settings) + - [Minio S3 settings](#minio-s3-settings) + - [Azure Blob Storage settings](#azure-blob-storage-settings) + - [DigitalOcean Spaces settings](#digitalocean-spaces-settings) + - [Openstack Object Storage settings](#openstack-object-storage-settings) + - [Integration testing](#integration-testing) + - [Applying core patches](#applying-core-patches) + - [Moodle 3.9:](#moodle-39) + - [Moodle 3.8:](#moodle-38) + - [Moodle 3.4 - 3.7:](#moodle-34---37) + - [Moodle 3.3 and Totara 12:](#moodle-33-and-totara-12) + - [Moodle 3.2 and Totara 11:](#moodle-32-and-totara-11) + - [Moodle 2.9 - 3.1 and Totara 2.9, 9 - 10:](#moodle-29---31-and-totara-29-9---10) + - [Moodle 2.7 - 2.8 and Totara 2.7 - 2.8:](#moodle-27---28-and-totara-27---28) + - [PHPUnit test compatibility](#phpunit-test-compatibility) + - [Contributing and support](#contributing-and-support) + - [Warm thanks](#warm-thanks) + - [Crafted by Catalyst IT](#crafted-by-catalyst-it) ## Use cases There are a number of different ways you can use this plug in. See [Recommended use case settings](#recommended-use-case-settings) for recommended settings for each one. @@ -75,7 +89,7 @@ This plugin is GDPR complient if you enable the deletion of remote objects. 3. Clone this repository into admin/tool/objectfs 4. Install one of the required SDK libraries for the storage file system that you will be using 1. Clone [moodle-local_aws](https://github.com/catalyst/moodle-local_aws) into local/aws for S3 or DigitalOcean Spaces or Google Cloud, or - 2. Clone [moodle-local_azure_storage](https://github.com/catalyst/moodle-local_azure_storage) into local/azure_storage for Azure Blob Storage, or + 2. Clone [moodle-local_azureblobstorage](https://github.com/catalyst/moodle-local_azureblobstorage) into local/azureblobstorage for Azure Blob Storage, or 3. Clone [moodle-local_openstack](https://github.com/matt-catalyst/moodle-local_openstack.git) into local/openstack for openstack(swift) storage 5. Install the plugins through the moodle GUI. 6. Configure the plugin. See [Moodle configuration](#moodle-configuration) diff --git a/TESTING.md b/TESTING.md index 2c20097c..2b59be30 100644 --- a/TESTING.md +++ b/TESTING.md @@ -15,7 +15,7 @@ $CFG->phpunit_objectfs_s3_integration_test_credentials = array( 's3_region' => 'Your region', ); ``` -* Azure: +* Azure (deprecated API): ```php $CFG->phpunit_objectfs_azure_integration_test_credentials = array( 'azure_accountname' => 'Your account name', @@ -23,6 +23,14 @@ $CFG->phpunit_objectfs_azure_integration_test_credentials = array( 'azure_sastoken' => 'Your sas token', ); ``` +* Azure Blob Storage: +```php +$CFG->phpunit_objectfs_azure_blob_storage_integration_test_credentials = [ + 'azure_accountname' => 'Your account name', + 'azure_container' => 'Your container', + 'azure_sastoken' => 'Your sas token', +]; +``` * Swift: ```php $CFG->phpunit_objectfs_swift_integration_test_credentials = array( diff --git a/classes/azure_blob_storage_file_system.php b/classes/azure_blob_storage_file_system.php new file mode 100644 index 00000000..3c798da4 --- /dev/null +++ b/classes/azure_blob_storage_file_system.php @@ -0,0 +1,33 @@ +. + +namespace tool_objectfs; + +use tool_objectfs\local\store\azure_blob_storage\file_system; + +/** + * File system for Azure Blob Storage. + * This file tells objectfs that this storage system is available for use. + * E.g. via $CFG->alternative_file_system_class + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class azure_blob_storage_file_system extends file_system { + +} diff --git a/classes/local/manager.php b/classes/local/manager.php index acd2b30e..ae763fda 100644 --- a/classes/local/manager.php +++ b/classes/local/manager.php @@ -306,17 +306,18 @@ public static function check_file_storage_filesystem() { public static function get_available_fs_list() { $result[''] = get_string('pleaseselect', OBJECTFS_PLUGIN_NAME); - $filesystems['\tool_objectfs\azure_file_system'] = '\tool_objectfs\azure_file_system'; + $filesystems['\tool_objectfs\azure_file_system'] = '\tool_objectfs\azure_file_system [DEPRECATED]'; $filesystems['\tool_objectfs\digitalocean_file_system'] = '\tool_objectfs\digitalocean_file_system'; $filesystems['\tool_objectfs\s3_file_system'] = '\tool_objectfs\s3_file_system'; $filesystems['\tool_objectfs\swift_file_system'] = '\tool_objectfs\swift_file_system'; + $filesystems['\tool_objectfs\azure_blob_storage_file_system'] = '\tool_objectfs\azure_blob_storage_file_system'; - foreach ($filesystems as $filesystem) { + foreach ($filesystems as $filesystem => $name) { $clientclass = self::get_client_classname_from_fs($filesystem); $client = new $clientclass(null); if ($client && $client->get_availability()) { - $result[$filesystem] = $filesystem; + $result[$filesystem] = $name; } } return $result; diff --git a/classes/local/store/azure/stream_wrapper.php b/classes/local/store/azure/stream_wrapper.php index 47a67a87..2fc7e7d9 100644 --- a/classes/local/store/azure/stream_wrapper.php +++ b/classes/local/store/azure/stream_wrapper.php @@ -36,6 +36,7 @@ use GuzzleHttp\Psr7\Stream; use GuzzleHttp\Psr7\CachingStream; use GuzzleHttp\Psr7; +use GuzzleHttp\Psr7\Utils; use MicrosoftAzure\Storage\Blob\BlobRestProxy; use MicrosoftAzure\Storage\Blob\Models\BlobProperties; use MicrosoftAzure\Storage\Blob\Models\SetBlobPropertiesOptions; @@ -443,7 +444,7 @@ private function openreadstream() { try { $blob = $client->getBlob($params['Container'], $params['Key']); - $this->body = Psr7\stream_for($blob->getContentStream()); + $this->body = Utils::streamFor($blob->getContentStream()); } catch (ServiceException $e) { // Prevent the client from keeping the request open when the content cannot be found. $response = $e->getResponse(); diff --git a/classes/local/store/azure_blob_storage/client.php b/classes/local/store/azure_blob_storage/client.php new file mode 100644 index 00000000..e67c5eee --- /dev/null +++ b/classes/local/store/azure_blob_storage/client.php @@ -0,0 +1,287 @@ +. + +namespace tool_objectfs\local\store\azure_blob_storage; + +use admin_settingpage; +use GuzzleHttp\Psr7\Utils; +use tool_objectfs\local\store\object_client_base; +use local_azureblobstorage\api; +use local_azureblobstorage\stream_wrapper; +use stdClass; +use Throwable; + +/** + * Azure blob storage client + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright 2024 Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class client extends object_client_base { + + /** @var api $api Azure API */ + protected api $api; + + /** + * Creates object client + * @param stdClass $config / TODO is this maybe null ? + */ + public function __construct($config) { + if (empty($config) || !$this->get_availability()) { + parent::__construct($config); + return; + } + + $this->api = new api($config->azure_accountname, $config->azure_container, $config->azure_sastoken); + $this->config = $config; + $this->maxupload = api::MAX_BLOCK_SIZE; + } + + /** + * Determines if this filesystem is available for use. + * @return bool + */ + public function get_availability(): bool { + // Requires local_azureblobstorage to be installed. + $info = \core\plugin_manager::instance()->get_plugin_info('local_azureblobstorage'); + + // Info is empty if plugin is not installed or no API is setup (missing config?). + return !empty($info); + } + + /** + * Sets the StreamWrapper to allow accessing the remote content via a blob:// path. + */ + public function register_stream_wrapper() { + if ($this->get_availability()) { + stream_wrapper::register($this->api); + } else { + parent::register_stream_wrapper(); + } + } + + /** + * Returns the full path for a given file by contenthash + * @param string $contenthash + * @return string filepath + */ + public function get_fullpath_from_hash($contenthash): string { + $filepath = $this->get_filepath_from_hash($contenthash); + return "blob://$filepath"; + } + + /** + * Returns the filepath from the contenthash, mimicking the + * structure of the filedir storage system. + * @param string $contenthash + * @return string filepath + */ + protected function get_filepath_from_hash($contenthash): string { + $l1 = $contenthash[0] . $contenthash[1]; + $l2 = $contenthash[2] . $contenthash[3]; + return "$l1/$l2/$contenthash"; + } + + /** + * Returns the blob key (the key used to reference the blob) from a given filepath. + * @param string $filepath + * @return string + */ + protected function get_blob_key_from_path(string $filepath): string { + return str_replace("blob://", '', $filepath); + } + + /** + * Deletes a given file + * @param string $fullpath + */ + public function delete_file($fullpath) { + // Stream wrapper supports unlinking, so just unlink. + unlink($fullpath); + } + + /** + * Renames a given file + * @param string $currentpath + * @param string $destinationpath + */ + public function rename_file($currentpath, $destinationpath) { + // Azure does not support renaming, instead the file is copied + // and the old one is deleted. + copy($currentpath, $destinationpath); + $this->delete_file($currentpath); + } + + /** + * Verifies an object is uploaded correctly. + * In Azure, this is done by checking the md5 hash of the contents. + * @param string $contenthash + * @param string $localpath + * @return bool + */ + public function verify_object($contenthash, $localpath) { + // If the object is uploaded to Azure the content will always be correct, + // because Azure will reject the original upload request if the md5 given during + // upload does not match. + // So here we just check the blob exists, and don't actually care about comparing the md5. + try { + // Just query the properties to confirm the file does indeed exist. + $key = $this->get_filepath_from_hash($contenthash); + $this->api->get_blob_properties_async($key)->wait(); + return true; + } catch (Throwable $e) { + return false; + } + } + + /** + * Returns a stream context used to handle file IO + * @return resource stream resource + */ + public function get_seekable_stream_context() { + $context = stream_context_create([ + 'blob' => [ + 'seekable' => true, + ], + ]); + return $context; + } + + /** + * Test permissions by uploading and doing various actions. + * @param bool $testdelete if should test deletion. + * @return stdClass containing 'success' and 'messages' values. + */ + public function test_permissions($testdelete): stdClass { + $key = 'permissions_check_test'; + $file = Utils::streamFor('test permission file'); + $filemd5 = hex2bin(md5('test permission file')); + + // Try create a file. + try { + $this->api->put_blob_async($key, $file, $filemd5)->wait(); + } catch (Throwable $e) { + return (object) [ + 'success' => false, + 'messages' => [get_string('settings:writefailure', 'tool_objectfs') . $e->getMessage() => 'notifyproblem'], + ]; + } + + // Try read the file that was created. + try { + $this->api->get_blob_async($key, $file, $filemd5)->wait(); + } catch (Throwable $e) { + return (object) [ + 'success' => false, + 'messages' => [get_string('settings:permissionreadfailure', 'tool_objectfs') . $e->getMessage() => 'notifyproblem'], + ]; + } + + // If testing delete, try delete the test file. + if ($testdelete) { + try { + $this->api->delete_blob_async($key)->wait(); + } catch (Throwable $e) { + return (object) [ + 'success' => false, + 'messages' => [get_string('settings:deleteerror', 'tool_objectfs') . $e->getMessage() => 'notifyproblem'], + ]; + } + } + + return (object) [ + 'success' => true, + 'messages' => [get_string('settings:permissioncheckpassed', 'tool_objectfs') => 'notifysuccess'], + ]; + } + + /** + * Tests connection + * @return stdClass with 'success' and 'details' values. + */ + public function test_connection(): stdClass { + // Try to create a file. + try { + $this->api->put_blob_async('connection_check_test', Utils::streamFor('test contents'), hex2bin(md5('test contents'))); + } catch (Throwable $e) { + return (object) [ + 'success' => false, + 'details' => $e->getMessage(), + ]; + } + + return (object) [ + 'success' => true, + 'details' => '', + ]; + } + + /** + * Returns token expiry time + * @return int + */ + public function get_token_expiry_time(): int { + if (empty($this->config->azure_sastoken)) { + return -1; + } + + // Parse the sas token (it just uses url parameter encoding). + $parts = []; + parse_str($this->config->azure_sastoken, $parts); + + // Get the 'se' part (signed expiry). + if (!isset($parts['se'])) { + // Assume expired (malformed). + return 0; + } + + // Parse timestamp string into unix timestamp int. + $expirystr = $parts['se']; + return strtotime($expirystr); + } + + /** + * Azure settings form with the following elements: + * + * Storage account name. + * Container name. + * Shared Access Signature. + * + * @param admin_settingpage $settings + * @param \stdClass $config + * @return admin_settingpage + */ + public function define_client_section($settings, $config): admin_settingpage { + $settings->add(new \admin_setting_heading('tool_objectfs/azure', + new \lang_string('settings:azure:header', 'tool_objectfs'), $this->define_client_check())); + + $settings->add(new \admin_setting_configtext('tool_objectfs/azure_accountname', + new \lang_string('settings:azure:accountname', 'tool_objectfs'), + new \lang_string('settings:azure:accountname_help', 'tool_objectfs'), '')); + + $settings->add(new \admin_setting_configtext('tool_objectfs/azure_container', + new \lang_string('settings:azure:container', 'tool_objectfs'), + new \lang_string('settings:azure:container_help', 'tool_objectfs'), '')); + + $settings->add(new \admin_setting_configpasswordunmask('tool_objectfs/azure_sastoken', + new \lang_string('settings:azure:sastoken', 'tool_objectfs'), + new \lang_string('settings:azure:sastoken_help', 'tool_objectfs'), '')); + + return $settings; + } +} diff --git a/classes/local/store/azure_blob_storage/file_system.php b/classes/local/store/azure_blob_storage/file_system.php new file mode 100644 index 00000000..275d58e2 --- /dev/null +++ b/classes/local/store/azure_blob_storage/file_system.php @@ -0,0 +1,39 @@ +. + +namespace tool_objectfs\local\store\azure_blob_storage; + +use tool_objectfs\local\store\azure_blob_storage\client; +use tool_objectfs\local\store\object_file_system; + +/** + * Azure blob store file system + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright 2024 Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class file_system extends object_file_system { + /** + * Initialise client + * @param mixed $config + * @return client + */ + protected function initialise_external_client($config) { + return new client($config); + } +} diff --git a/classes/tests/test_azure_blob_storage_integration_client.php b/classes/tests/test_azure_blob_storage_integration_client.php new file mode 100644 index 00000000..6f8804b8 --- /dev/null +++ b/classes/tests/test_azure_blob_storage_integration_client.php @@ -0,0 +1,60 @@ +. + +namespace tool_objectfs\tests; + +use tool_objectfs\local\store\azure_blob_storage\client; + +/** + * Client used for integration testing azure blob storage client + * + * @package tool_objectfs + * @author Matthew Hilton + * @copyright 2024 Catalyst IT + * @license http://www.gnu.org/copyleft/gpl.html GNU GPL v3 or later + */ +class test_azure_blob_storage_integration_client extends client { + /** + * @var string + */ + private $runidentifier; + + /** + * construct + * @param mixed $config + * @return void + */ + public function __construct($config) { + parent::__construct($config); + $time = microtime(); + $this->runidentifier = md5($time); + } + + /** + * get_filepath_from_hash + * @param mixed $contenthash + * + * @return string + */ + protected function get_filepath_from_hash($contenthash): string { + $l1 = $contenthash[0] . $contenthash[1]; + $l2 = $contenthash[2] . $contenthash[3]; + $runidentifier = $this->runidentifier; + return "test/$runidentifier/$l1/$l2/$contenthash"; + } + +} + diff --git a/classes/tests/test_file_system.php b/classes/tests/test_file_system.php index 7e1ba875..2a938035 100644 --- a/classes/tests/test_file_system.php +++ b/classes/tests/test_file_system.php @@ -64,6 +64,13 @@ protected function initialise_external_client($config) { $config->azure_sastoken = $credentials['azure_sastoken']; manager::set_objectfs_config($config); $client = new test_azure_integration_client($config); + } else if (isset($CFG->phpunit_objectfs_azure_blob_storage_integration_test_credentials)) { + $credentials = $CFG->phpunit_objectfs_azure_integration_test_credentials; + $config->azure_accountname = $credentials['azure_accountname']; + $config->azure_container = $credentials['azure_container']; + $config->azure_sastoken = $credentials['azure_sastoken']; + manager::set_objectfs_config($config); + $client = new test_azure_blob_storage_integration_client($config); } else if (isset($CFG->phpunit_objectfs_swift_integration_test_credentials)) { $credentials = $CFG->phpunit_objectfs_swift_integration_test_credentials; $config->openstack_authurl = $credentials['openstack_authurl']; diff --git a/tests/object_file_system_test.php b/tests/object_file_system_test.php index 6843bd6b..0abe42a7 100644 --- a/tests/object_file_system_test.php +++ b/tests/object_file_system_test.php @@ -945,6 +945,13 @@ public function test_is_configured_fake_autoloader() { $autoloaderref = $clientref->getParentClass()->getProperty('autoloader'); $autoloaderref->setAccessible(true); $autoloader = $autoloaderref->getValue($this->filesystem->externalclient); + + // If client does not have autoloader, skip test. + if (empty($autoloader)) { + $this->markTestSkipped("Client does not have autoloader"); + return; + } + $this->set_externalclient_config('autoloader', $autoloader . '_fake'); $this->assertFalse($this->filesystem->is_configured()); }