From a80a98ef37e36a397cf0bbd9b5266536eb85ac45 Mon Sep 17 00:00:00 2001 From: DrunkBatya Date: Mon, 18 Dec 2023 03:33:57 -0500 Subject: [PATCH] Add uploading files without reindex, minor fixes --- Dockerfile | 13 +++--- README.md | 7 ++++ indexer/main.py | 10 ++++- indexer/src/file_upload.py | 44 ++++++++++++++++++-- indexer/src/repository.py | 2 + indexer/src/settings.py | 2 +- nginx/nginx.conf | 2 +- indexer/requirements.txt => requirements.txt | 0 8 files changed, 69 insertions(+), 11 deletions(-) rename indexer/requirements.txt => requirements.txt (100%) diff --git a/Dockerfile b/Dockerfile index cb517cf..20580be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,13 +1,16 @@ FROM python:3.11-alpine3.17 +RUN apk update +RUN apk add tzdata nginx-mod-http-fancyindex nginx-mod-http-headers-more bash + +ADD requirements.txt /app/ +RUN python3 -m pip install -r /app/requirements.txt + COPY nginx/nginx.conf /etc/nginx/nginx.conf COPY nginx/nginx-theme /var/lib/nginx/html/nginx-theme ADD indexer /app COPY startup.sh /app/ -WORKDIR /app -RUN apk update -RUN apk add tzdata nginx-mod-http-fancyindex nginx-mod-http-headers-more bash -RUN python3 -m pip install -r requirements.txt +WORKDIR /app +CMD ["/bin/bash", "/app/startup.sh"] -CMD ["/bin/bash", "startup.sh"] diff --git a/README.md b/README.md index 5b859d1..e2b4c1a 100644 --- a/README.md +++ b/README.md @@ -47,3 +47,10 @@ Upload files -F "files=@flipper-z-f7-full-0.73.1.json" \ 127.0.0.1:8000/firmware/uploadfiles ``` + +Upload files without reindex +```bash + curl -L -H "Token: YOUR_TOKEN" \ + -F "files=@gcc-arm-none-eabi-12.3-arm64-darwin-flipper-24.tar.gz" \ + 127.0.0.1:8000/toolchain/uploadfilesraw +``` diff --git a/indexer/main.py b/indexer/main.py index 14dc459..d6a41c0 100755 --- a/indexer/main.py +++ b/indexer/main.py @@ -5,7 +5,7 @@ import uvicorn from fastapi import FastAPI, Request, Response from src import directories, file_upload, security -from src.directories import indexes +from src.repository import indexes, raw_file_upload_directories from src.settings import settings from pygelf import GelfTcpHandler @@ -25,9 +25,17 @@ def startup_event() -> None: os.makedirs(settings.files_dir) for index in indexes: try: + index_path = os.path.join(settings.files_dir, index) + os.makedirs(index_path, exist_ok=True) indexes[index].reindex() except Exception: logging.exception(f"Init {index} reindex failed") + for raw_upload_dir in raw_file_upload_directories: + try: + dir_path = os.path.join(settings.files_dir, raw_upload_dir) + os.makedirs(dir_path, exist_ok=True) + except Exception: + logging.exception(f"Failed to create {dir_path}") app.include_router(file_upload.router) diff --git a/indexer/src/file_upload.py b/indexer/src/file_upload.py index 79da4be..d83f767 100644 --- a/indexer/src/file_upload.py +++ b/indexer/src/file_upload.py @@ -7,7 +7,7 @@ from typing import List from fastapi import APIRouter, Form, UploadFile from fastapi.responses import JSONResponse -from .directories import indexes +from .repository import indexes, raw_file_upload_directories from .settings import settings @@ -46,7 +46,7 @@ def save_files(path: str, files: List[UploadFile]) -> None: out_file.write(file.file.read()) -def move_files(dest_dir: str, source_dir: str, version_token: str) -> None: +def move_files_for_indexed(dest_dir: str, source_dir: str, version_token: str) -> None: token_file_path = os.path.join(dest_dir, TOKEN_FILENAME) do_cleanup = False if version_token and os.path.isfile(token_file_path): @@ -67,6 +67,13 @@ def move_files(dest_dir: str, source_dir: str, version_token: str) -> None: shutil.move(sourcefilepath, destfilepath) +def move_files_raw(dest_dir: str, source_dir: str) -> None: + for file in os.listdir(source_dir): + sourcefilepath = os.path.join(source_dir, file) + destfilepath = os.path.join(dest_dir, file) + shutil.move(sourcefilepath, destfilepath) + + @router.post("/{directory}/uploadfiles") async def create_upload_files( directory: str, @@ -101,7 +108,7 @@ async def create_upload_files( try: with tempfile.TemporaryDirectory() as temp_path: save_files(temp_path, files) - move_files(final_path, temp_path, version_token) + move_files_for_indexed(final_path, temp_path, version_token) logging.info(f"Uploaded {len(files)} files") except Exception as e: logging.exception(e) @@ -117,3 +124,34 @@ async def create_upload_files( ) else: return JSONResponse("File uploaded, reindexing isn't needed!") + + +@router.post("/{directory}/uploadfilesraw") +async def create_upload_files_raw( + directory: str, + files: List[UploadFile], +): + """ + A method to upload files in a certain directory without indexing + Args: + directory: Repository name + files: File list + + Returns: + Upload status + """ + if directory not in raw_file_upload_directories: + return JSONResponse(f"{directory} not found!", status_code=404) + + project_root_path = os.path.join(settings.files_dir, directory) + + async with lock: + try: + with tempfile.TemporaryDirectory() as temp_path: + save_files(temp_path, files) + move_files_raw(project_root_path, temp_path) + logging.info(f"Uploaded {len(files)} files") + return JSONResponse("File uploaded") + except Exception as e: + logging.exception(e) + return JSONResponse(str(e), status_code=500) diff --git a/indexer/src/repository.py b/indexer/src/repository.py index 74af8a3..a7872db 100644 --- a/indexer/src/repository.py +++ b/indexer/src/repository.py @@ -155,3 +155,5 @@ def get_file_from_latest_version( file_parser=blackmagicFileParser, ), } + +raw_file_upload_directories = ["toolchain"] diff --git a/indexer/src/settings.py b/indexer/src/settings.py index 66c028f..13c72c9 100644 --- a/indexer/src/settings.py +++ b/indexer/src/settings.py @@ -44,5 +44,5 @@ class Settings(BaseModel): qFlipper_github_repo=os.getenv("INDEXER_QFLIPPER_GITHUB_REPO"), blackmagic_github_token=os.getenv("INDEXER_BLACKMAGIC_GITHUB_TOKEN"), blackmagic_github_repo=os.getenv("INDEXER_BLACKMAGIC_GITHUB_REPO"), - private_paths=["reindex", "uploadfiles"], + private_paths=["reindex", "uploadfiles", "uploadfilesraw"], ) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index f4084e9..c7c11f7 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -37,7 +37,7 @@ http { fancyindex_localtime on; fancyindex_ignore "nginx-theme"; } - location ~ ^/(qFlipper|firmware|blackmagic-firmware)/ { + location ~ ^/(qFlipper|firmware|blackmagic-firmware|toolchain)/ { more_set_headers 'Cache-Control: no-cache, max-age=0, s-max-age=0, no-store, must-revalidate, max-stale=0, post-check=0, pre-check=0'; proxy_set_header Host $host; proxy_set_header X-Real-IP $remote_addr; diff --git a/indexer/requirements.txt b/requirements.txt similarity index 100% rename from indexer/requirements.txt rename to requirements.txt