-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
0 parents
commit 31fd5bb
Showing
24 changed files
with
3,390 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
**/__pycache__ | ||
**/.venv | ||
**/.classpath | ||
**/.dockerignore | ||
**/.env | ||
**/.git | ||
**/.gitignore | ||
**/.project | ||
**/.settings | ||
**/.toolstarget | ||
**/.vs | ||
**/.vscode | ||
**/*.*proj.user | ||
**/*.dbmdl | ||
**/*.jfm | ||
**/bin | ||
**/charts | ||
**/docker-compose* | ||
**/compose* | ||
**/Dockerfile* | ||
**/node_modules | ||
**/npm-debug.log | ||
**/obj | ||
**/secrets.dev.yaml | ||
**/values.dev.yaml | ||
LICENSE |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,133 @@ | ||
# Byte-compiled / optimized / DLL files | ||
__pycache__/ | ||
*.py[cod] | ||
*$py.class | ||
|
||
# C extensions | ||
*.so | ||
|
||
# Distribution / packaging | ||
.Python | ||
env/ | ||
build/ | ||
develop-eggs/ | ||
dist/ | ||
downloads/ | ||
eggs/ | ||
.eggs/ | ||
lib/ | ||
lib64/ | ||
parts/ | ||
sdist/ | ||
var/ | ||
wheels/ | ||
*.egg-info/ | ||
.installed.cfg | ||
*.egg | ||
|
||
# PyInstaller | ||
# Usually these files are written by a python script from a template | ||
# before PyInstaller builds the exe, so as to inject date/other infos into it. | ||
*.manifest | ||
*.spec | ||
|
||
# Installer logs | ||
pip-log.txt | ||
pip-delete-this-directory.txt | ||
|
||
# Unit test / coverage reports | ||
htmlcov/ | ||
.tox/ | ||
.coverage | ||
.coverage.* | ||
.cache | ||
nosetests.xml | ||
coverage.xml | ||
*.cover | ||
.hypothesis/ | ||
|
||
# Translations | ||
*.mo | ||
*.pot | ||
|
||
# Django stuff: | ||
*.log | ||
local_settings.py | ||
|
||
# Flask stuff: | ||
instance/ | ||
.webassets-cache | ||
|
||
# Scrapy stuff: | ||
.scrapy | ||
|
||
# Sphinx documentation | ||
docs/_build/ | ||
|
||
# PyBuilder | ||
target/ | ||
|
||
# Jupyter Notebook | ||
.ipynb_checkpoints | ||
|
||
# pyenv | ||
.python-version | ||
|
||
# celery beat schedule file | ||
celerybeat-schedule | ||
|
||
# SageMath parsed files | ||
*.sage.py | ||
|
||
# dotenv | ||
.env | ||
|
||
# virtualenv | ||
.venv | ||
venv/ | ||
ENV/ | ||
|
||
# Spyder project settings | ||
.spyderproject | ||
.spyproject | ||
|
||
# Rope project settings | ||
.ropeproject | ||
|
||
# mkdocs documentation | ||
/site | ||
|
||
# mypy | ||
.mypy_cache/ | ||
node_modules | ||
|
||
|
||
tags | ||
cscope.out | ||
cscope.files | ||
|
||
# idea | ||
.idea/ | ||
|
||
# dependencies | ||
node_modules/ | ||
|
||
# testing | ||
/coverage | ||
|
||
# production | ||
build/ | ||
|
||
# misc | ||
npm-debug.log* | ||
yarn-debug.log* | ||
yarn-error.log* | ||
.DS_Store | ||
|
||
# pytest | ||
.pytest_cache/ | ||
|
||
# ruff | ||
.ruff_cache | ||
|
||
.vscode/ |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,15 @@ | ||
repos: | ||
- repo: local | ||
hooks: | ||
- id: black | ||
name: Black | ||
entry: /bin/sh -c 'cd ./proxy/ && black .' | ||
language: system | ||
types: [python] | ||
- id: ruff | ||
stages: [commit] | ||
name: Ruff | ||
types: [python] | ||
language: system | ||
entry: /bin/sh -c 'cd ./proxy/ && ruff check . --config ../pyproject.toml --show-source --fix' | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
FROM pypy:3.10 as prod | ||
|
||
ARG INSTALL_ADDONS=true | ||
|
||
ENV APP_HOME=/app | ||
ENV POETRY_OPTS= | ||
|
||
RUN pip install -U poetry | ||
|
||
RUN mkdir $APP_HOME | ||
|
||
COPY ./proxy $APP_HOME/proxy | ||
COPY ./README.md $APP_HOME/README.md | ||
COPY ./plugin_hooks/* $APP_HOME/proxy/ | ||
COPY poetry.lock pyproject.toml $APP_HOME | ||
|
||
WORKDIR $APP_HOME | ||
|
||
# install optionally required libraries | ||
RUN [[ ${INSTALL_ADDONS} = "true" ]] || exit 0 && \ | ||
apt-get update && \ | ||
apt-get install -y libmagic1 | ||
|
||
RUN poetry config virtualenvs.create false && if [ "$INSTALL_ADDONS" = "true" ]; then poetry install --with addons --without dev; else poetry install --only main; fi | ||
|
||
|
||
EXPOSE 8000 | ||
|
||
CMD ["poetry", "run", "python", "-m", "proxy.app"] | ||
|
||
FROM prod as dev_image | ||
|
||
# rust is required for aiohttp-devtools | ||
RUN curl https://sh.rustup.rs -sSf | bash -s -- -y | ||
ENV PATH="/root/.cargo/bin:${PATH}" | ||
|
||
RUN poetry install | ||
|
||
CMD ["adev", "runserver", "proxy"] | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
# Object storage encryption proxy | ||
|
||
## Pluggable Hooks | ||
|
||
The proxy forwards all traffic to the configured S3-store. | ||
|
||
When uploading and downloading you can inject hooks in three categories: | ||
`pre_upload_before_check` that will be called before upload simultaneously | ||
`pre_upload_unsafe` that will be called before upload but after | ||
`pre_upload_before_check` has run successfully. You can run calls on the content | ||
here that will actually interpret the data, assuming the previous call has checked | ||
for malignous data. | ||
`post_upload` will run after the upload has run successfully. It will not change the | ||
existing state. Consider it side-effects. | ||
|
||
Hooks are named with the function name by default. This can be overriden when register- | ||
ing. | ||
|
||
The default hooks for encryption/decryption are `hook_encrypt_data` and | ||
`hook_decrypt_data` respectively. If you override the hooks in your implementation | ||
you should import them from the `proxy.default_hooks` module. You may of course | ||
override those, too. | ||
|
||
Every hook should take the current request plus optionally data or other params and | ||
return a tuple of the name, boolen of success and optionally resulting data or | ||
the error message on failure. | ||
|
||
Per event all registered hooks must pass in order to proceed. | ||
To register a procedure as a hook it has to be wrapped in a hook-function that takes | ||
the request and binary data with the execution position. If the execution position is | ||
irrelevant `pos` can be omitted or set to `0` and will thus be incremented accordingly. | ||
|
||
Unless overridden hooks will be read from `proxy.default_hooks`. Hooks are overridden | ||
by placing decorated functions in the `plugin_hooks/hooks.py` file. If you want to | ||
keep the default encryption mechanism import the `pre_upload_before_checks` event from | ||
`proxy.default_hooks`. If you wish to override it import the event from `proxy.events`. | ||
|
||
The `pos` parameter for registering hooks is only relevant if the registered hooks | ||
should be called in a predefined order. | ||
|
||
|
||
## Incldued addon dependencies | ||
|
||
The `INSTALL_ADDONS` build arg (true/false) controls whether to install the following extra dependencies (default: true) | ||
|
||
- `python-clamd` | ||
- `python-magic` | ||
|
||
|
||
|
||
## start the development server | ||
|
||
``` | ||
docker compose up -d --build | ||
``` | ||
|
||
This will pull up a local instance on :8080 | ||
|
||
It will also create a minio service on :9000 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,31 @@ | ||
version: '3.4' | ||
|
||
services: | ||
proxy: | ||
image: s3-hooked/proxy:dev | ||
build: | ||
target: dev_image | ||
environment: | ||
- PROXY_SECRET=dqxNYNhp9nfnYQjpUE2dzGnXBHem3shYd0lIaNKuyomem6SB | ||
- PROXY_ENVIRONMENT=${ENVIRONMENT:-development} | ||
- MINIO_ACCESS_KEY=minio-dev-key | ||
- MINIO_SECRET_KEY=5hA9J2SW5syVch0u5oJtVwUJokR7loHHYh1rMV8V | ||
- PROXY_OBJECT_STORE_HOST=minio | ||
- PROXY_OBJECT_STORE_PORT=9000 | ||
- PROXY_OBJECT_STORE_SSL_ENABLED=false | ||
- PYTHONBREAKPOINT=ipdb.set_trace | ||
volumes: | ||
- .:/app/ | ||
ports: | ||
- "8080:8000" | ||
command: adev runserver proxy | ||
|
||
minio: | ||
environment: | ||
- MINIO_ROOT_USER=admin | ||
- MINIO_ROOT_PASSWORD=minio-admin | ||
- MINIO_BROWSER=on | ||
ports: | ||
- "9000:9000" | ||
- "9090:9090" | ||
command: server data --console-address :9090 |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
version: '3.4' | ||
|
||
services: | ||
proxy: | ||
build: | ||
dockerfile: Dockerfile | ||
context: . | ||
target: prod | ||
args: | ||
- INSTALL_ADDONS=true | ||
image: s3-hooked/proxy:prod | ||
environment: | ||
- PROXY_SECRET | ||
- PROXY_OBJECT_STORE_HOST=${PROXYO_OBJECT_STORE_HOST:-minio} | ||
- PROXY_OBJECT_STORE_PORT=${PROXYO_OBJECT_STORE_PORT:-9000} | ||
- PROXY_OBJECT_STORE_SSL_ENABLED=${PROXY_OJECT_STORE_SSL_ENABLED:-true} | ||
ports: | ||
- "8080:8000" | ||
minio: | ||
image: minio/minio:RELEASE.2023-09-16T01-01-47Z | ||
volumes: | ||
- minio_data:/data | ||
command: server data | ||
|
||
volumes: | ||
minio_data: | ||
|
Empty file.
Oops, something went wrong.