CLI: add 'rest' command

pyunicore/cli/base.py

@@ -201,3 +201,92 @@ def show_token_details(self, token: str):
         print(f"Issued by:    {payload['iss']}")
         print(f"Valid for:    {payload.get('aud', '<unlimited>')}")
         print(f"Renewable:    {payload.get('renewable', 'no')}")
+
+
+class REST(Base):
+    def add_command_args(self):
+        self.parser.prog = "unicore rest"
+        self.parser.description = self.get_synopsis()
+        self.parser.add_argument("command", help="Operation (GET, PUT, POST, DELETE)")
+        self.parser.add_argument("URL", help="Endpoint URL(s)", nargs="*")
+        self.parser.add_argument(
+            "-A",
+            "--accept",
+            required=False,
+            default="application/json",
+            help="Value for the 'Accept' header (default: 'application/json')",
+        )
+        self.parser.add_argument(
+            "-C",
+            "--content-type",
+            required=False,
+            default="application/json",
+            help="Value for the 'Content-Type' header (default: 'application/json')",
+        )
+        self.parser.add_argument(
+            "-i",
+            "--include",
+            required=False,
+            action="store_true",
+            help="Include the response headers in the output",
+        )
+        self.parser.add_argument("-D", "--data", required=False, help="JSON data for PUT/POST")
+
+    def get_synopsis(self):
+        return """Low-level REST API operations."""
+
+    def get_description(self):
+        return "perform a low-level REST API operation"
+
+    def get_group(self):
+        return "Utilities"
+
+    def run(self, args):
+        super().setup(args)
+        cmd = self.args.command
+        if not cmd:
+            raise ValueError("REST operation (GET, PUT, POST, DELETE) required.")
+        _headers = {"Accept": self.args.accept, "Content-Type": self.args.content_type}
+        for endpoint in self.args.URL:
+            self.verbose(f"HTTP {cmd} {endpoint}")
+            self.to_json = "application/json" in self.args.accept
+            tr = pyunicore.client.Transport(credential=self.credential, use_security_sessions=False)
+            if "GET".startswith(cmd.upper()):
+                response = tr.get(to_json=False, url=endpoint, headers=_headers)
+            elif "POST".startswith(cmd.upper()):
+                response = tr.post(url=endpoint, headers=_headers, data=self._read_data())
+            elif "PUT".startswith(cmd.upper()):
+                response = tr.put(url=endpoint, headers=_headers, data=self._read_data())
+            elif "DELETE".startswith(cmd.upper()):
+                response = tr.delete(url=endpoint)
+            else:
+                raise ValueError("Unsupported operation '%s'" % cmd)
+            if response:
+                self._print_response(response)
+
+    def _print_response(self, response, print_body=True):
+        print(f"HTTP/1.1 {response.status_code} {response.reason}")
+        if self.args.include:
+            self._print_headers(response)
+        if response.headers.get("Location"):
+            self._last_location = response.headers["Location"]
+            print(self._last_location)
+        if print_body and response.status_code not in [201, 204]:
+            if self.to_json:
+                print(json.dumps(response.json(), indent=2))
+            else:
+                print(response.content)
+
+    def _print_headers(self, response):
+        for h in response.headers:
+            print(f"{h}: {response.headers[h]}")
+
+    def _read_data(self):
+        if self.args.data:
+            d = self.args.data
+            if d.startswith("@"):
+                with open(d[1:]) as f:
+                    d = f.read()
+            return d
+        else:
+            return None

pyunicore/cli/io.py

@@ -102,7 +102,7 @@ def _download(self, source_endpoint, source_path, target_path):
                 have_stdout = True
                 target = os.fdopen(sys.stdout.fileno(), "wb", closefd=False)
             elif os.path.isdir(target_path):
-                target = normalized(target_path + "/" + os.path.basename(fname))
+                target = normalized(target_path + "/" + basename(fname))
             else:
                 target = target_path
             self.verbose(f"... {source_endpoint}/files{fname} -> {target}")
@@ -124,18 +124,42 @@ def run(self, args):
         target_endpoint, target_path = self.parse_location(self.args.target)
         for s in self.args.source:
             source_endpoint, source_path = self.parse_location(s)
-            if len(self.args.source) > 1:
-                target = target_path + "/" + basename(source_path)
+            if source_endpoint is not None:
+                self._download(source_endpoint, source_path, target_path)
             else:
-                target = target_path
-                if target.endswith("/"):
-                    target = target + basename(source_path)
-                if target.endswith("."):
-                    target = target + "/" + basename(source_path)
+                self._upload(source_path, target_endpoint, target_path)
+
+
+class Cat(IOBase):
+    def add_command_args(self):
+        self.parser.prog = "unicore cat"
+        self.parser.description = self.get_synopsis()
+        self.parser.add_argument("source", nargs="+", help="Source(s)")
+
+    def get_synopsis(self):
+        return """Prints remote file(s) to standard output"""
+
+    def get_description(self):
+        return "cat remote files"
+
+    def _cat(self, source_endpoint, source_path):
+        storage = Storage(self.credential, storage_url=source_endpoint)
+        base_dir, file_pattern = split_path(source_path)
+        for fname in crawl_remote(storage, base_dir, file_pattern):
+            p = storage.stat(fname)
+            target = os.fdopen(sys.stdout.fileno(), "wb", closefd=False)
+            self.verbose(f"... {source_endpoint}/files{fname}")
+            p.download(target)
+            target.close()
+
+    def run(self, args):
+        super().setup(args)
+        for s in self.args.source:
+            source_endpoint, source_path = self.parse_location(s)
             if source_endpoint is not None:
-                self._download(source_endpoint, source_path, target)
+                self._cat(source_endpoint, source_path)
             else:
-                self._upload(source_path, target_endpoint, target)
+                raise ValueError("Not a remote file: %s" % s)
 
 
 def normalized(path: str):

pyunicore/cli/main.py

@@ -9,12 +9,14 @@
 
 _commands = {
     "cancel-job": pyunicore.cli.exec.CancelJob,
+    "cat": pyunicore.cli.io.Cat,
     "cp": pyunicore.cli.io.CP,
     "exec": pyunicore.cli.exec.Exec,
     "issue-token": pyunicore.cli.base.IssueToken,
     "job-status": pyunicore.cli.exec.GetJobStatus,
     "list-jobs": pyunicore.cli.exec.ListJobs,
     "ls": pyunicore.cli.io.LS,
+    "rest": pyunicore.cli.base.REST,
     "run": pyunicore.cli.exec.Run,
 }
 

pyunicore/client.py

@@ -204,8 +204,8 @@ def post(self, **kwargs):
         return self.run_method(requests.post, **kwargs)
 
     def delete(self, **kwargs):
-        """send a DELETE to the current endpoint"""
-        self.run_method(requests.delete, **kwargs).close()
+        """send a DELETE to the current endpoint and return the response"""
+        return self.run_method(requests.delete, **kwargs)
 
 
 class Resource:
@@ -256,7 +256,7 @@ def links(self):
 
     def delete(self):
         """delete/destroy this resource"""
-        self.transport.delete(url=self.resource_url)
+        self.transport.delete(url=self.resource_url).close()
 
     def set_properties(self, props):
         """set/update resource properties"""
@@ -756,15 +756,15 @@ def mkdir(self, name):
 
     def rmdir(self, name):
         """remove a directory and all its content"""
-        return self.transport.delete(url=self._to_file_url(name))
+        self.transport.delete(url=self._to_file_url(name)).close()
 
     def rm(self, name):
         """remove a file"""
-        return self.transport.delete(url=self._to_file_url(name))
+        self.transport.delete(url=self._to_file_url(name)).close()
 
     def makedirs(self, name):
         """create directory"""
-        return self.mkdir(name)
+        self.mkdir(name)
 
     def upload(self, file_name, destination=None):
         """upload local file "file_name" to the remote file "destination".

tests/integration/cli/test_base.py

@@ -20,6 +20,27 @@ def test_issue_token(self):
         args = ["-c", config_file, ep, "--lifetime", "700", "--inspect", "--limited", "--renewable"]
         cmd.run(args)
 
+    def test_rest_get(self):
+        cmd = base.REST()
+        config_file = "tests/integration/cli/preferences"
+        ep = "https://localhost:8080/DEMO-SITE/rest/core"
+        args = ["-c", config_file, "GET", ep]
+        cmd.run(args)
+
+    def test_rest_post_put_delete(self):
+        cmd = base.REST()
+        config_file = "tests/integration/cli/preferences"
+        ep = "https://localhost:8080/DEMO-SITE/rest/core/sites"
+        d = "{}"
+        args = ["-c", config_file, "--data", d, "POST", ep]
+        cmd.run(args)
+        d = "{'tags': 'test123'}"
+        ep = cmd._last_location
+        args = ["-c", config_file, "--data", d, "PUT", ep]
+        cmd.run(args)
+        args = ["-c", config_file, "DELETE", ep]
+        cmd.run(args)
+
 
 if __name__ == "__main__":
     unittest.main()