Merge branch 'binpash:main' into adding-systems

infrastructure/.gitignore

@@ -1,3 +1,4 @@
 __pycache__/
 .venv
 venv
+target/shellmetrics.sh

infrastructure/Makefile

@@ -1,4 +1,4 @@
-STATIC_OUTPUTS = target/lines_of_code.csv target/nodes_in_scripts.csv target/scripts_to_benchmark.csv
+STATIC_OUTPUTS = target/lines_of_code.csv target/nodes_in_scripts.csv target/scripts_to_benchmark.csv target/cyclomatic.csv target/shellmetrics.sh
 
 static: $(STATIC_OUTPUTS)
 
@@ -17,6 +17,13 @@ static-test: tests/test_syntax_analysis.py
 clean-static: 
 	rm -f $(STATIC_OUTPUTS)
 
+target/shellmetrics.sh:
+	wget --quiet -O $@ https://raw.githubusercontent.com/shellspec/shellmetrics/b3bfff2af6880443112cdbf2ea449440b30ab9b0/shellmetrics
+	chmod +x $@
+
+target/cyclomatic.csv: get_cyclomatic.py target/shellmetrics.sh
+	python3 get_cyclomatic.py > $@
+
 dynamic:
 
 .PHONY: static dynamic clean-static static-test

infrastructure/get_cyclomatic.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+from collections import defaultdict
+from pathlib import Path
+from typing import Optional
+import json
+from subprocess import check_output
+
+from all_scripts import get_all_scripts
+from syntax_analysis import parse_shell_script, count_nodes
+from project_root import get_project_root
+
+root = get_project_root()
+shellmetrics = root / 'infrastructure' / 'target' / 'shellmetrics.sh'
+if not shellmetrics.is_file():
+    raise f'shellmetrics.sh not in `{shellmetrics}`, please put the script at that path'
+
+all_scripts = [s for scripts in get_all_scripts().values() for s in scripts]
+
+output = check_output([shellmetrics, '--csv', '--shell', 'bash', '--no-color', *all_scripts], text=True)
+datas = defaultdict(list)
+for line in output.splitlines()[1:]:
+    file, _func, _lineno, _lloc, ccn, _lines, _comment, _blank = line.split(',')
+    file = json.loads(file)
+    file = Path(file).relative_to(root)
+    datas[file].append((ccn))
+
+for file, datas in datas.items():
+    ccn = sum(float(ccn) for ccn, in datas)
+    print(file, ccn, sep=',')

infrastructure/target/cyclomatic.csv

@@ -0,0 +1,88 @@
+covid-mts/scripts/1.sh,1.0
+covid-mts/scripts/3.sh,1.0
+covid-mts/scripts/4.sh,1.0
+covid-mts/scripts/2.sh,1.0
+covid-mts/scripts/5.sh,1.0
+file-enc/scripts/encrypt_files.sh,3.0
+file-enc/scripts/compress_files.sh,2.0
+log-analysis/scripts/nginx.sh,3.0
+log-analysis/scripts/pcaps.sh,3.0
+max-temp/scripts/temp-analytics.sh,1.0
+media-conv/scripts/to_mp3.sh,3.0
+media-conv/scripts/img_convert.sh,3.0
+nlp/scripts/verses_2om_3om_2instances.sh,2.0
+nlp/scripts/trigram_rec.sh,3.0
+nlp/scripts/uppercase_by_token.sh,2.0
+nlp/scripts/words_no_vowels.sh,2.0
+nlp/scripts/sort_words_by_folding.sh,2.0
+nlp/scripts/syllable_words_2.sh,2.0
+nlp/scripts/count_morphs.sh,2.0
+nlp/scripts/sort_words_by_num_of_syllables.sh,3.0
+nlp/scripts/merge_upper.sh,2.0
+nlp/scripts/syllable_words_1.sh,2.0
+nlp/scripts/sort_words_by_rhyming.sh,2.0
+nlp/scripts/letter_words.sh,2.0
+nlp/scripts/sort.sh,2.0
+nlp/scripts/count_consonant_seq.sh,2.0
+nlp/scripts/count_trigrams.sh,3.0
+nlp/scripts/bigrams.sh,3.0
+nlp/scripts/count_words.sh,2.0
+nlp/scripts/vowel_sequencies_gr_1K.sh,2.0
+nlp/scripts/bigrams_appear_twice.sh,3.0
+nlp/scripts/compare_exodus_genesis.sh,3.0
+nlp/scripts/uppercase_by_type.sh,2.0
+nlp/scripts/find_anagrams.sh,3.0
+nlp/scripts/count_vowel_seq.sh,2.0
+oneliners/scripts/spell.sh,1.0
+oneliners/scripts/wf.sh,1.0
+oneliners/scripts/bi-gram.aux.sh,4.0
+oneliners/scripts/top-n.sh,1.0
+oneliners/scripts/sort-sort.sh,1.0
+oneliners/scripts/diff.sh,1.0
+oneliners/scripts/sort.sh,1.0
+oneliners/scripts/bi-grams.sh,1.0
+oneliners/scripts/set-diff.sh,1.0
+oneliners/scripts/nfa-regex.sh,1.0
+sklearn/scripts/run.sh,1.0
+riker/scripts/lua/build.sh,1.0
+riker/scripts/xz-clang/build.sh,1.0
+riker/scripts/vim/build.sh,1.0
+riker/scripts/xz/build.sh,1.0
+riker/scripts/redis/build.sh,1.0
+riker/scripts/sqlite/build.sh,1.0
+riker/scripts/memcached/build.sh,2.0
+uniq-ips/scripts/run.sh,1.0
+unix50/scripts/1.sh,2.0
+unix50/scripts/10.sh,2.0
+unix50/scripts/16.sh,2.0
+unix50/scripts/21.sh,2.0
+unix50/scripts/19.sh,2.0
+unix50/scripts/6.sh,2.0
+unix50/scripts/36.sh,2.0
+unix50/scripts/28.sh,2.0
+unix50/scripts/12.sh,2.0
+unix50/scripts/13.sh,2.0
+unix50/scripts/26.sh,2.0
+unix50/scripts/31.sh,2.0
+unix50/scripts/24.sh,2.0
+unix50/scripts/11.sh,2.0
+unix50/scripts/23.sh,2.0
+unix50/scripts/14.sh,2.0
+unix50/scripts/25.sh,2.0
+unix50/scripts/7.sh,2.0
+unix50/scripts/29.sh,2.0
+unix50/scripts/30.sh,2.0
+unix50/scripts/3.sh,2.0
+unix50/scripts/20.sh,2.0
+unix50/scripts/4.sh,2.0
+unix50/scripts/34.sh,2.0
+unix50/scripts/2.sh,2.0
+unix50/scripts/18.sh,2.0
+unix50/scripts/33.sh,2.0
+unix50/scripts/35.sh,2.0
+unix50/scripts/32.sh,2.0
+unix50/scripts/15.sh,2.0
+unix50/scripts/17.sh,2.0
+unix50/scripts/9.sh,2.0
+unix50/scripts/8.sh,2.0
+unix50/scripts/5.sh,2.0

oneliners/scripts/run.sh

@@ -0,0 +1 @@
+cat "logs-popcount-org.txt" | sort | uniq > "out.txt"

uniq-ips/run.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+REPO_TOP=$(git rev-parse --show-toplevel)
+eval_dir="${REPO_TOP}/uniq-ips"
+scripts_dir="${eval_dir}/scripts"
+
+BENCHMARK_SHELL=${BENCHMARK_SHELL:-bash}
+cd "$eval_dir" # scripts/run.sh puts files in its current directory
+$BENCHMARK_SHELL "$scripts_dir/run.sh" $@
+