forked from openedx-unsupported/edx-analytics-pipeline
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathMakefile
137 lines (101 loc) · 6.47 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
.PHONY: requirements test test-requirements .tox upgrade
uninstall:
while pip uninstall -y edx.analytics.tasks; do true; done
python setup.py clean
install: requirements uninstall
python setup.py install --force
bootstrap: uninstall
pip install -r requirements/pip.txt
pip install -r requirements/base.txt --no-cache-dir
python setup.py install --force
develop: requirements develop-local
develop-local: uninstall
python setup.py develop
python setup.py install_data
docker-pull:
docker pull edxops/analytics_pipeline:latest
docker-shell:
docker run -v `(pwd)`:/edx/app/analytics_pipeline/analytics_pipeline -it edxops/analytics_pipeline:latest bash
system-requirements:
ifeq (,$(wildcard /usr/bin/yum))
# This is not great, we can't use these libraries on slave nodes using this method.
sudo apt-get install -y -q libmysqlclient-dev libpq-dev python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev
else
sudo yum install -y -q postgresql-devel libffi-devel
endif
requirements:
pip install -r requirements/pip.txt
pip install -r requirements/default.txt --no-cache-dir
pip install -r requirements/extra.txt --no-cache-dir
test-requirements: requirements
pip install -r requirements/test.txt --no-cache-dir
reset-virtualenv:
# without bash, environment variables are not available
bash -c 'virtualenv --clear ${ANALYTICS_PIPELINE_VENV}/analytics_pipeline'
upgrade: ## update the requirements/*.txt files with the latest packages satisfying requirements/*.in
pip install -qr requirements/pip-tools.txt
CUSTOM_COMPILE_COMMAND="make upgrade" pip-compile --upgrade -o requirements/pip-tools.txt requirements/pip-tools.in
CUSTOM_COMPILE_COMMAND="make upgrade" pip-compile --upgrade -o requirements/base.txt requirements/base.in
CUSTOM_COMPILE_COMMAND="make upgrade" pip-compile --upgrade -o requirements/default.txt requirements/default.in
CUSTOM_COMPILE_COMMAND="make upgrade" pip-compile --upgrade -o requirements/docs.txt requirements/docs.in
CUSTOM_COMPILE_COMMAND="make upgrade" pip-compile --upgrade -o requirements/test.txt requirements/test.in
test-docker-local:
docker run --rm -u root -v `(pwd)`:/edx/app/analytics_pipeline/analytics_pipeline -it edxops/analytics_pipeline:latest make develop-local test-local
test-docker:
docker run --rm -u root -v `(pwd)`:/edx/app/analytics_pipeline/analytics_pipeline -it edxops/analytics_pipeline:latest make reset-virtualenv test-requirements develop-local test-local
test-local:
# TODO: when we have better coverage, modify this to actually fail when coverage is too low.
rm -rf .coverage
LUIGI_CONFIG_PATH='config/test.cfg' python -m coverage run --rcfile=./.coveragerc -m nose --with-xunit --xunit-file=unittests.xml -A 'not acceptance'
test: test-requirements develop test-local
test-acceptance: test-requirements
LUIGI_CONFIG_PATH='config/test.cfg' python -m coverage run --rcfile=./.coveragerc -m nose --nocapture --with-xunit -A acceptance $(ONLY_TESTS)
test-acceptance-local:
REMOTE_TASK=$(shell which remote-task) LUIGI_CONFIG_PATH='config/test.cfg' ACCEPTANCE_TEST_CONFIG="/var/tmp/acceptance.json" python -m coverage run --rcfile=./.coveragerc -m nose --nocapture --with-xunit -A acceptance --stop -v $(ONLY_TESTS)
test-acceptance-local-all:
REMOTE_TASK=$(shell which remote-task) LUIGI_CONFIG_PATH='config/test.cfg' ACCEPTANCE_TEST_CONFIG="/var/tmp/acceptance.json" python -m coverage run --rcfile=./.coveragerc -m nose --nocapture --with-xunit -A acceptance -v
quality-docker:
docker run --rm -u root -v `(pwd)`:/edx/app/analytics_pipeline/analytics_pipeline -it edxops/analytics_pipeline:latest isort --check-only --recursive edx/
docker run --rm -u root -v `(pwd)`:/edx/app/analytics_pipeline/analytics_pipeline -it edxops/analytics_pipeline:latest pycodestyle edx
coverage-docker:
docker run --rm -u root -v `(pwd)`:/edx/app/analytics_pipeline/analytics_pipeline -it edxops/analytics_pipeline:latest coverage xml
coverage-local: test-local
python -m coverage html
python -m coverage xml -o coverage.xml
diff-cover coverage.xml --html-report diff_cover.html
isort --check-only --recursive edx/
# Compute pep8 quality
diff-quality --violations=pycodestyle --html-report diff_quality_pep8.html
pycodestyle edx > pep8.report || echo "Not pep8 clean"
# Compute pylint quality
pylint -f parseable edx --msg-template "{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" > pylint.report || echo "Not pylint clean"
diff-quality --violations=pylint --html-report diff_quality_pylint.html pylint.report
coverage: test coverage-local
docs-requirements:
pip install -r requirements/pip.txt
pip install -r requirements/docs.txt --no-cache-dir
python setup.py install --force
docs-local:
sphinx-build -b html docs/source docs
docs-clean:
rm -rf docs/*.* docs/_*
docs: docs-requirements docs-local
todo:
pylint --disable=all --enable=W0511 edx
# for docker devstack
docker-test-acceptance-local:
LAUNCH_TASK=$(shell which launch-task) REMOTE_TASK=$(shell which remote-task) LUIGI_CONFIG_PATH='config/docker_test.cfg' ACCEPTANCE_TEST_CONFIG="/edx/etc/edx-analytics-pipeline/acceptance.json" python -m coverage run --rcfile=./.coveragerc -m nose --nocapture --with-xunit -A acceptance --stop -v $(ONLY_TESTS)
docker-test-acceptance-local-all:
LAUNCH_TASK=$(shell which launch-task) REMOTE_TASK=$(shell which remote-task) LUIGI_CONFIG_PATH='config/docker_test.cfg' ACCEPTANCE_TEST_CONFIG="/edx/etc/edx-analytics-pipeline/acceptance.json" python -m coverage run --rcfile=./.coveragerc -m nose --nocapture --with-xunit -A acceptance -v
generate-spark-egg-files:
# edx-opaque-keys egg
mkdir -p /var/tmp/edx_egg_files/
curl -fSL https://files.pythonhosted.org/packages/63/86/d4bf9c7e7a720125b0572c43dda002f72be2cc66313be601cd7b6cccb2ad/edx-opaque-keys-0.4.tar.gz -o /var/tmp/edx-opaque-keys.tar.gz
cd /var/tmp/ && mkdir -p /var/tmp/edx-opaque-keys/ && tar --strip-components=1 -xzf /var/tmp/edx-opaque-keys.tar.gz -C /var/tmp/edx-opaque-keys/
cd /var/tmp/edx-opaque-keys && python setup.py bdist_egg
cp /var/tmp/edx-opaque-keys/dist/edx_opaque_keys-0.4-py2.7.egg /var/tmp/edx_egg_files/edx_opaque_keys.egg
# edx-ccx-keys agg
curl -fSL https://files.pythonhosted.org/packages/a4/03/444d30a3859e36ef2273ae6254c011e1fb3a02f4e3dba16008ecf0b4bdb3/edx-ccx-keys-0.2.1.tar.gz -o /var/tmp/edx-ccx-keys.tar.gz
cd /var/tmp/ && mkdir -p /var/tmp/edx-ccx-keys/ && tar --strip-components=1 -xzf /var/tmp/edx-ccx-keys.tar.gz -C /var/tmp/edx-ccx-keys/
cd /var/tmp/edx-ccx-keys && python setup.py bdist_egg
cp /var/tmp/edx-ccx-keys/dist/edx_ccx_keys-0.2.1-py2.7.egg /var/tmp/edx_egg_files/edx_ccx_keys.egg