hecat/Makefile at master · nodiscc/hecat · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
SHELL := /bin/bash

.PHONY: help # generate list of targets with descriptions
help:
	@echo "USAGE: make TARGET"
	@echo "Available targets:"
	@grep '^.PHONY: .* #' Makefile | sed 's/\.PHONY: \(.*\) # \(.*\)/\1	\2/' | expand -t20

.PHONY: clean # clean files generated by make install/test_run
clean:
	-rm -rf build/ dist/ hecat.egg-info/ tests/awesome-selfhosted tests/awesome-selfhosted-data tests/audio/ tests/video/ tests/shaarli.yml tests/html-table hecat.log tests/awesome-selfhosted-html tests/requirements.txt trivy trivy_*_Linux-64bit.tar.gz tests/webpages/

# do not install sphinx from setup.py/install_requires, workaround for https://github.com/sphinx-doc/sphinx/issues/11130
.PHONY: install # install in a virtualenv
install:
	python3 -m venv .venv && source .venv/bin/activate && \
	pip3 install wheel && \
	python3 -m pip install .

##### TESTS #####

.PHONY: test # run tests
test: test_pylint clean test_import_shaarli test_download_video test_download_audio test_export_html_table clone_awesome_selfhosted test_import_awesome_selfhosted test_process_awesome_selfhosted test_awesome_lint test_export_awesome_selfhosted_md test_export_awesome_selfhosted_html test_archive_webpages scan_trivy

test_short: test_pylint clean test_import_shaarli test_archive_webpages test_download_video test_download_audio test_export_html_table clone_awesome_selfhosted test_awesome_lint test_export_awesome_selfhosted_md test_export_awesome_selfhosted_html

.PHONY: test # run all tests
test: test_short test_long

.PHONY: test_short # run tests except those that consume github API requests/long URL checks
test_short: clean test_import_shaarli test_archive_webpages test_download_video test_download_audio test_export_html_table \
    clone_awesome_selfhosted test_export_awesome_selfhosted_md test_awesome_lint \
    test_export_awesome_selfhosted_html

.PHONY: test_short # run long tests
test_long: test_process_awesome_selfhosted

.PHONY: test_pylint # run linter (non blocking)
test_pylint: install
	.venv/bin/pip3 install pylint pyyaml
	.venv/bin/pylint --fail-on E --fail-under=9.45 --disable=too-many-locals,line-too-long,consider-using-f-string,no-else-return hecat

.PHONY: clone_awesome_selfhosted # clone awesome-selfhosted/awesome-selfhosted-data
clone_awesome_selfhosted:
	git clone --depth=1 https://github.com/awesome-selfhosted/awesome-selfhosted tests/awesome-selfhosted
	git clone https://github.com/awesome-selfhosted/awesome-selfhosted-data tests/awesome-selfhosted-data

.PHONY: test_import_awesome_selfhosted # test import from awesome-sefhosted
test_import_awesome_selfhosted: install
	rm -rf tests/awesome-selfhosted-data/{tags,software,platforms}
	mkdir tests/awesome-selfhosted-data/{tags,software,platforms}
	source .venv/bin/activate && \
	hecat --config tests/.hecat.import_awesome_selfhosted.yml && \
	hecat --config tests/.hecat.import_awesome_selfhosted_nonfree.yml

.PHONY: test_process_awesome_selfhosted # test all processing steps on awesome-selfhosted-data
test_process_awesome_selfhosted: install test_url_check test_update_software_metadata test_awesome_lint
	cd tests/awesome-selfhosted-data && git --no-pager diff --color=always

.PHONY: test_url_check # test URL checker on awesome-sefhosted-data
test_url_check: install
	source .venv/bin/activate && \
	hecat --config tests/.hecat.url_check.yml

.PHONY: test_update_software_metadata # test software metadata updater/processor on awesome-selfhosted-data
test_update_software_metadata: install
	source .venv/bin/activate && \
	hecat --log-level DEBUG --config tests/.hecat.update_metadata.yml

.PHONY: test_awesome_lint # test linter/compliance checker on awesome-sefhosted-data
test_awesome_lint: install
	source .venv/bin/activate && \
	hecat --config tests/.hecat.awesome_lint.yml

.PHONY: test_export_awesome_selfhosted_md # test export to singlepage markdown from awesome-selfhosted-data
test_export_awesome_selfhosted_md: install
	source .venv/bin/activate && \
	hecat --config tests/.hecat.export_markdown_singlepage.yml && \
	cd tests/awesome-selfhosted && git --no-pager diff --color=always

.PHONY: test_export_awesome_selfhosted_html # test export to singlepage HTML from awesome-selfhosted-data
test_export_awesome_selfhosted_html: install
	rm -rf tests/awesome-selfhosted-html
	mkdir -p tests/awesome-selfhosted-html
	source .venv/bin/activate && \
	hecat --config tests/.hecat.export_markdown_multipage.yml && \
	sed -i 's|<a href="https://github.com/pradyunsg/furo">Furo</a>|<a href="https://github.com/nodiscc/hecat/">hecat</a>, <a href="https://www.sphinx-doc.org/">sphinx</a> and <a href="https://github.com/pradyunsg/furo">furo</a>. Content under <a href="https://github.com/awesome-selfhosted/awesome-selfhosted-data/blob/master/LICENSE">CC-BY-SA 3.0</a> license.|' .venv/lib/python*/site-packages/furo/theme/furo/page.html && \
	sphinx-build -b html -c tests/ -d tests/awesome-selfhosted-html/.doctrees tests/awesome-selfhosted-html/md/ tests/awesome-selfhosted-html/html/
	# remove unused files for static site publication
	rm tests/awesome-selfhosted-html/html/.buildinfo tests/awesome-selfhosted-html/html/objects.inv

.PHONY: test_import_shaarli # test import from shaarli JSON
test_import_shaarli: install
	source .venv/bin/activate && \
	hecat --config tests/.hecat.import_shaarli.yml

.PHONY: test_download_video # test downloading videos from the shaarli import, test log file creation
test_download_video: install
	rm -f tests/hecat.log
	source .venv/bin/activate && \
	hecat --log-file tests/hecat.log --config tests/.hecat.download_video.yml
	grep -q 'writing data file tests/shaarli.yml' tests/hecat.log

.PHONY: test_download_audio # test downloading audio files from the shaarli import
test_download_audio: install
	source .venv/bin/activate && \
	hecat --config tests/.hecat.download_audio.yml

.PHONY: test_archive_webpages # test webpage archiving
test_archive_webpages: install
	mkdir -p tests/webpages
	# create directory that should be removed by clean_removed
	mkdir -p tests/webpages/public/9999999999
	source .venv/bin/activate && \
	hecat --log-level DEBUG --config tests/.hecat.archive_webpages.yml
	# test existence and content of archived page
	@grep -q 'official Debian installation media can be purchased' tests/webpages/public/232/www.debian.org/releases/stable/amd64/ch01s05.en.html
	# test that directories were effectively removed
	@if [[ -d tests/webpages/public/9999999999 ]]; then echo "ERROR tests/webpages/public/9999999999 should have been removed by clean_removed: True"; exit 1; fi
	@if [[ -d tests/webpages/public/6625 ]]; then echo "ERROR tests/webpages/public/6625 should have been removed by clean_excluded: True"; exit 1; fi


.PHONY: test_export_html_table # test exporting shaarli data to HTML table
test_export_html_table: install
	mkdir -p tests/html-table
	source .venv/bin/activate && \
	hecat --config tests/.hecat.export_html_table.yml

TRIVY_VERSION=0.44.0
TRIVY_EXIT_CODE=1
.PHONY: scan_trivy # run trivy vulnerability scanner
scan_trivy:
	source .venv/bin/activate && pip3 freeze --local > tests/requirements.txt
	wget --quiet --continue -O trivy_$(TRIVY_VERSION)_Linux-64bit.tar.gz https://github.com/aquasecurity/trivy/releases/download/v$(TRIVY_VERSION)/trivy_$(TRIVY_VERSION)_Linux-64bit.tar.gz
	tar -z -x trivy -f trivy_$(TRIVY_VERSION)_Linux-64bit.tar.gz
	./trivy --exit-code $(TRIVY_EXIT_CODE) fs tests/requirements.txt