Herbie eval / artifact

scripts/oopsla23/herbie/.gitignore

@@ -0,0 +1,4 @@
+/herbie
+/bench
+/reports
+/rules

scripts/oopsla23/herbie/README.md

@@ -0,0 +1,80 @@
+## Herbie evaluation
+
+Herbie evaluation for OOPSLA 2023
+
+### Dependencies
+
+To run this evaluation, the following dependencies must be installed:
+ - Bash
+ - Python3
+ - Racket (>= 8.2)
+ - Rust (>= 1.60)
+
+Optionally, install:
+ - GNU Parallel (needed for `env_parallel`)
+
+### Components
+
+Scripts:
+- `eval.sh`: top-level script for running the evaluation
+- `install.sh`: installs Herbie under `herbie/`
+- `plot.sh`: top-level plotting script
+- `seed-variance.sh`: script for running Herbie, possibly in parallel
+
+Plotting:
+- `plot/config-all-tests-box-plot.py`: generates plots
+- `plot/plot-results.sh`: preprocesses evaluation data for plotting
+
+Directories:
+- `bench`: directory where benchmarks are collated for evaluation
+- `herbie`: directory where Herbie is installed
+- `plot`: plotting code stored under here
+- `reports`: Herbie reports generated here
+- `saved`: A previous run for use in the artifact
+- `rules`: Ruler rules need to be stored here to run the evaluation
+
+### Results from Previous Runs
+
+Generating the results of
+  this experiment takes exceptionally long.
+For the evaluation of this paper,
+  we ran this experiment on a machine with 512 GB of RAM
+  utilizing 60 cores at a time, and it took 6 hours
+  to complete.
+We recommend running: `scripts/oopsla23/plot.sh`:
+
+```
+cd scripts/oopsla23/herbie
+./plot.sh saved/
+```
+
+to generate the results
+  from a previous run that has be
+  included with this VM.
+It should take no more than 20 seconds
+  and should emit .png and .pdf files under `saved/`.
+
+### Usage
+
+This WILL take a long time to run.
+Running a single seed takes approximately 45 minutes
+  and requires 4 cores and at least 8 GB of RAM.
+
+First, run Ruler to generate a
+  unified `*.json` file containing the rules
+  from all the domains it knows about.
+Copy or move that file to `rules/output.json`.
+
+To generate Herbie data from scratch:
+```
+bash eval.sh <num seeds>
+```
+  where the evaluation runs Herbie over `<num seeds>` seeds
+  over the same benchmark set.
+For the evaluation we used 30 seeds.
+Optionally,
+  set the environment variable `PARALLEL_SEEDS`
+  to run Herbie in parallel (This requires `env_parallel`.)
+When the evaluation is finished,
+  the plots will be stored in a timestamped folder
+  under `reports/`.

scripts/oopsla23/herbie/eval.sh

@@ -0,0 +1,129 @@
+#!/usr/bin/env bash
+
+##
+## Runs the Herbie eval
+## Requires clone access to remote Herbie repo https://github.com/herbie-fp/herbie
+## Requires rules to be generated and placed under `rules/`
+##
+
+# exit immediately upon first error
+set -e
+
+# determine physical directory of this script
+src="${BASH_SOURCE[0]}"
+while [ -L "$src" ]; do
+  dir="$(cd -P "$(dirname "$src")" && pwd)"
+  src="$(readlink "$src")"
+  [[ $src != /* ]] && src="$dir/$src"
+done
+
+MYDIR="$(cd -P "$(dirname "$src")" && pwd)"
+
+# Constants
+HERBIE_DIR="$MYDIR/herbie"
+BENCH_DIR="$MYDIR/bench"
+THREADS=4   # 4 is a good number for consistent performance
+FLAGS="-o generate:taylor --timeout 300"
+GROUP="main"
+RULES="$MYDIR/rules/output.json"
+
+if [ -z "$1" ]; then
+  echo "expected a number of seeds to run"
+  exit 1
+else
+  NUM_SEEDS="$1"
+fi
+
+echo "Running with $NUM_SEEDS seeds"
+
+# Output
+tstamp="$(date "+%Y-%m-%d_%H%M")"
+OUTDIR="$MYDIR/reports/$tstamp"
+
+#
+#   Install Herbie
+#
+
+BUILD_DIR=$HERBIE_DIR bash install.sh
+
+#
+#   Make benchmark set
+#
+
+rm -rf $BENCH_DIR
+mkdir -p $BENCH_DIR
+
+cp -r "$HERBIE_DIR/bench/hamming" \
+      "$HERBIE_DIR/bench/mathematics" \
+      "$HERBIE_DIR/bench/numerics" \
+      "$HERBIE_DIR/bench/physics" \
+      "$HERBIE_DIR/bench/pbrt.fpcore" \
+      "$BENCH_DIR/"
+
+FPCORES=$(find $BENCH_DIR -name "*.fpcore")
+for fpcore in $FPCORES; do
+  echo "filtering $fpcore"
+  racket "$MYDIR/filter.rkt" \
+    --names "raw-angle from scale-rotated-ellipse ; \
+             a from scale-rotated-ellipse ; \
+             b from scale-rotated-ellipse ; \
+             Simplification of discriminant from scale-rotated-ellipse ; \
+             Distance on a great circle ; \
+             Harley's example" \
+    $fpcore >> "$fpcore.tmp"
+  mv "$fpcore.tmp" $fpcore
+done
+
+#
+#   Run the branches!!!
+#
+
+function do_branch {
+  name="$1"; shift
+  flags="$@"
+
+  # Prepare run
+  pushd $HERBIE_DIR
+
+  # Generate rules if necessary
+  git checkout $HERBIE_DIR/src/syntax/rules.rkt
+  if [[ "$name" != "main" ]]; then
+    racket $MYDIR/gen-rules.rkt $name $RULES $MYDIR/rules.rkt
+    mv $MYDIR/rules.rkt $HERBIE_DIR/src/syntax/rules.rkt
+  fi
+
+  # Patch egg
+  cp "$MYDIR/lib.rs" "$HERBIE_DIR/egg-herbie/src/lib.rs"
+
+  # Install
+  make install
+  popd
+
+  # Run the seed survey
+  if [ -z "$NO_RUN" ]; then
+    HERBIE=$HERBIE_DIR \
+      PARALLEL_SEEDS=$PARALLEL_SEEDS \
+      THREADS=$THREADS \
+      BENCH=$BENCH_DIR \
+      HERBIE_FLAGS="$FLAGS $flags" \
+      bash seed-variance.sh $NUM_SEEDS "$OUTDIR/$name" $name
+  fi
+}
+
+# Run each configuration
+
+do_branch main
+do_branch enumo
+do_branch enumo-ruler-rat
+# do_branch enumo-replicate-rat
+do_branch enumo-no-ff
+do_branch enumo-rat
+do_branch ruler
+
+#
+#   Plots
+#
+
+if [ -z "$NO_RUN" ]; then
+  bash plot.sh $OUTDIR
+fi

scripts/oopsla23/herbie/filter.rkt

@@ -0,0 +1,79 @@
+#lang racket
+
+;; script for filtering out the benchmarks in Herbie
+
+(define (ops-in-expr expr)
+  (define ops (mutable-set))
+  (let loop ([expr expr])
+    (match expr
+     [(list 'let (list (list vars vals) ...) body)
+      (set-add! ops 'let)
+      (for ([val vals]) (loop val))
+      (loop body)]
+     [(list 'let* (list (list vars vals) ...) body)
+      (set-add! ops 'let*)
+      (for ([val vals]) (loop val))
+      (loop body)]
+     [(list op args ...)
+      (set-add! ops op)
+      (for ([arg args]) (loop arg))]
+     [_ (void)]))
+  (map symbol->string (set->list ops)))
+
+(define (syntax-e* stx)
+  (match (syntax-e stx)
+   [(list xs ...) (map syntax-e* xs)]
+   [x x]))
+
+(define (make-pairs l acc)
+  (if (= 0 (modulo (length l) 2))
+      (cond [(empty? l) acc]
+            [else (make-pairs (rest (rest l)) (cons (cons (first l) (first (rest l))) acc))])
+      (raise "The list is not pair-able (uneven length)")))
+
+(define (load-file file ops names invert?)
+  (define names* (apply set names))
+  (call-with-input-file file
+    (λ (port)
+      (port-count-lines! port)
+      (for ([core (in-port (curry read-syntax file) port)])
+        (define core* (syntax-e* core))
+        (match-define (list 'FPCore vars props ... body) core*)
+        (define in-expr (ops-in-expr body))
+        (define ps (make-hash (make-pairs props '())))
+        (define nm (string-trim (hash-ref ps ':name)))
+        (cond
+          [(not (set-member? names* nm))
+           (pretty-print core* (current-output-port) 1)
+           (newline)]
+          [else
+           (eprintf "Removed ~s\n" nm)])))))
+
+(define (load-directory dir ops names invert?)
+  (for ([fname (in-directory dir)]
+        #:when (file-exists? fname)
+        #:when (equal? (filename-extension fname) #"fpcore"))
+    (load-file fname ops names invert?)))
+
+(define (filter-cores path ops names invert?)
+  (define path* (if (string? path) (string->path path) path))
+  (define out
+    (cond
+     [(directory-exists? path*)
+      (load-directory path* ops names invert?)]
+     [else
+      (load-file path* ops names invert?)]))
+  (void))
+
+(module+ main
+ (define invert? #f)
+ (define ops '())
+ (define names '())
+ (command-line
+  #:program "filter.rkt"
+  #:once-each
+  ["--names" strs "Names of bad fp-cores (; separated single string)"
+    (set! names (map (lambda (x) (string-trim x)) (string-split strs ";")))]
+  #:args paths
+ (for ([path paths]) (filter-cores path ops names invert?))
+ (void)))