1#!/bin/bash 2# 3# End-to-end tests for the dashboard. 4# 5# Usage: 6# ./regtest.sh <function name> 7# 8# NOTE: Must be run in this directory (rappor/pipeline). 9 10set -o nounset 11set -o pipefail 12set -o errexit 13 14# Create schema and params. 15create-metadata() { 16 mkdir -p _tmp/metadata 17 echo 'Hello from regtest.sh' 18 19 local params_path=_tmp/metadata/regtest_params.csv 20 21 # Relying on $RAPPOR_SRC/regtest.sh 22 cp --verbose ../_tmp/python/demo1/case_params.csv $params_path 23 24 # For now, use the same map everywhere. 25 cat >_tmp/metadata/dist-analysis.csv <<EOF 26var,map_filename 27unif,map.csv 28gauss,map.csv 29exp,map.csv 30m.domain,domain_map.csv 31EOF 32 33 # Both single dimensional and multi dimensional metrics. 34 cat >_tmp/metadata/rappor-vars.csv <<EOF 35metric,var,var_type,params 36m,domain,string,m_params 37m,flag..HTTPS,boolean,m_params 38unif,,string,regtest_params 39gauss,,string,regtest_params 40exp,,string,regtest_params 41EOF 42} 43 44# Create map files. 45create-maps() { 46 mkdir -p _tmp/maps 47 # Use the same map for everyone now? 48 local map_path=_tmp/maps/map.csv 49 50 # Relying on $RAPPOR_SRC/regtest.sh 51 cp --verbose ../_tmp/python/demo1/case_map.csv $map_path 52} 53 54# Simulate different metrics. 55create-counts() { 56 mkdir -p _tmp/counts 57 58 for date in 2015-12-01 2015-12-02 2015-12-03; do 59 mkdir -p _tmp/counts/$date 60 61 # TODO: Change params for each day. 62 cp --verbose \ 63 ../_tmp/python/demo1/1/case_counts.csv _tmp/counts/$date/unif_counts.csv 64 cp --verbose \ 65 ../_tmp/python/demo2/1/case_counts.csv _tmp/counts/$date/gauss_counts.csv 66 cp --verbose \ 67 ../_tmp/python/demo3/1/case_counts.csv _tmp/counts/$date/exp_counts.csv 68 done 69} 70 71dist-task-spec() { 72 local job_dir=$1 73 ./task_spec.py dist \ 74 --map-dir _tmp/maps \ 75 --config-dir _tmp/metadata \ 76 --output-base-dir $job_dir/raw \ 77 --bad-report-out _tmp/bad_counts.csv \ 78 "$@" 79} 80 81dist-job() { 82 local job_id=$1 83 local pat=$2 84 85 local job_dir=_tmp/$job_id 86 mkdir -p $job_dir/raw 87 88 local spec_list=$job_dir/spec-list.txt 89 90 find _tmp/counts/$pat -name \*_counts.csv \ 91 | dist-task-spec $job_dir \ 92 | tee $spec_list 93 94 ./dist.sh decode-dist-many $job_dir $spec_list 95 ./dist.sh combine-and-render-html _tmp $job_dir 96} 97 98dist() { 99 create-metadata 100 create-maps 101 create-counts 102 103 dist-job smoke1 '2015-12-01' # one day 104 dist-job smoke2 '2015-12-0[23]' # two days 105} 106 107# Simulate different metrics. 108create-reports() { 109 mkdir -p _tmp/reports 110 111 for date in 2015-12-01 2015-12-02 2015-12-03; do 112 mkdir -p _tmp/reports/$date 113 114 # TODO: Change params for each day. 115 cp --verbose \ 116 ../bin/_tmp/reports.csv _tmp/reports/$date/m_reports.csv 117 done 118} 119 120assoc-task-spec() { 121 local job_dir=$1 122 ./task_spec.py assoc \ 123 --map-dir _tmp/maps \ 124 --config-dir _tmp/metadata \ 125 --output-base-dir $job_dir/raw \ 126 "$@" 127} 128 129assoc-job() { 130 local job_id=$1 131 local pat=$2 132 133 local job_dir=_tmp/$job_id 134 mkdir -p $job_dir/raw $job_dir/config 135 136 local spec_list=$job_dir/spec-list.txt 137 138 find _tmp/reports/$pat -name \*_reports.csv \ 139 | assoc-task-spec $job_dir \ 140 | tee $spec_list 141 142 # decode-many calls decode_assoc.R, which expects this schema in the 'config' 143 # dir now. TODO: adjust this. 144 cp --verbose _tmp/metadata/rappor-vars.csv $job_dir/config 145 cp --verbose ../bin/_tmp/m_params.csv $job_dir/config 146 147 ./assoc.sh decode-many $job_dir $spec_list 148 ./assoc.sh combine-and-render-html _tmp $job_dir 149} 150 151# Copy some from bin/test.sh? The input _reports.csv files should be taken 152# from there. 153assoc() { 154 create-reports 155 cp --verbose ../bin/_tmp/domain_map.csv _tmp/maps 156 157 assoc-job smoke1-assoc '2015-12-01' # one day 158 assoc-job smoke2-assoc '2015-12-0[23]' # two days 159} 160 161"$@" 162