1#!/bin/bash 2# 3# Take the raw data from the analysis and massage it into various formats 4# suitable for display. 5# 6# Usage: 7# ./cook.sh <function name> 8 9set -o nounset 10set -o pipefail 11set -o errexit 12 13readonly THIS_DIR=$(dirname $0) 14readonly RAPPOR_SRC=$(cd $THIS_DIR/.. && pwd) 15 16source $RAPPOR_SRC/pipeline/tools-lib.sh 17 18 19status-files() { 20 local dir=$1 21 find $dir -name STATUS.txt 22} 23 24results-files() { 25 local dir=$1 26 find $dir -name results.csv 27} 28 29count-results() { 30 # first field of each line is one of {OK, TIMEOUT, FAIL, SKIPPED} 31 status-files "$@" \ 32 | xargs cat \ 33 | cut -d ' ' -f 1 \ 34 | sort | uniq -c | sort -n -r 35} 36 37# 38# For dist cron job 39# 40 41# Combine status of tasks over multiple jobs. Each row is a task (decode-dist 42# invocation). This has the number of reports. 43combine-dist-task-status() { 44 local base_dir=${1:-~/rappor/cron} 45 local job_dir=${2:-~/rappor/cron/2015-05-22__05-58-01} 46 47 local out=$job_dir/task-status.csv 48 49 # Ignore memory for now. 50 time status-files $base_dir | TOOLS-combine-status dist > $out 51 echo "Wrote $out" 52} 53 54# Create a single dist.csv time series for a GIVEN metric. 55combine-dist-results-one() { 56 local base_dir=$1 57 local job_dir=$2 58 local metric_name=$3 59 #echo FOO $base_dir $metric_name 60 61 local out_dir=$job_dir/cooked/$metric_name 62 mkdir -p $out_dir 63 64 # Glob to capture this specific metric name over ALL job IDs. 65 find $base_dir/*/raw/$metric_name -name STATUS.txt \ 66 | TOOLS-combine-results dist 5 \ 67 > $out_dir/dist.csv 68} 69 70# Creates a dist.csv file for EACH metric. TODO: Rename one/many 71combine-dist-results() { 72 local base_dir=${1:-~/rappor/cron} 73 local job_dir=${2:-~/rappor/cron/2015-05-22__05-58-01} 74 75 # Direct subdirs of 'raw' are metrics. Just print filename. 76 find $base_dir/*/raw -mindepth 1 -maxdepth 1 -type d -a -printf '%f\n' \ 77 | sort | uniq \ 78 | xargs --verbose -n1 -- \ 79 $0 combine-dist-results-one $base_dir $job_dir 80} 81 82# Take the task-status.csv file, which has row key (metric, date). Writes 83# num_reports.csv and status.csv per metric, and a single overview.csv for all 84# metrics. 85dist-metric-status() { 86 local job_dir=${1:-_tmp/results-10} 87 local out_dir=$job_dir/cooked 88 89 TOOLS-metric-status dist $job_dir/task-status.csv $out_dir 90} 91 92# 93# For association analysis cron job 94# 95 96combine-assoc-task-status() { 97 local base_dir=${1:-~/rappor/chrome-assoc-smoke} 98 local job_dir=${2:-$base_dir/smoke1} 99 100 local out=$job_dir/assoc-task-status.csv 101 102 time find $base_dir -name assoc-status.txt \ 103 | TOOLS-combine-status assoc \ 104 > $out 105 106 echo "Wrote $out" 107} 108 109# Create a single assoc.csv time series for a GIVEN (var1, var2) pair. 110combine-assoc-results-one() { 111 local base_dir=$1 112 local job_dir=$2 113 local metric_pair_rel_path=$3 114 115 local out_dir=$job_dir/cooked/$metric_pair_rel_path 116 mkdir -p $out_dir 117 118 # Glob to capture this specific metric name over ALL job IDs. 119 find $base_dir/*/raw/$metric_pair_rel_path -name assoc-status.txt \ 120 | TOOLS-combine-results assoc 5 \ 121 > $out_dir/assoc-results-series.csv 122} 123 124# Creates a dist.csv file for EACH metric. TODO: Rename one/many 125combine-assoc-results() { 126 local base_dir=${1:-~/rappor/chrome-assoc-smoke} 127 local job_dir=${2:-$base_dir/smoke3} 128 129 # Direct subdirs of 'raw' are metrics, and subdirs of that are variable 130 # pairs. Print "$metric_name/$pair_name". 131 find $base_dir/*/raw -mindepth 2 -maxdepth 2 -type d -a -printf '%P\n' \ 132 | sort | uniq \ 133 | xargs --verbose -n1 -- \ 134 $0 combine-assoc-results-one $base_dir $job_dir 135} 136 137# Take the assoc-task-status.csv file, which has row key (metric, date). Writes 138# num_reports.csv and status.csv per metric, and a single overview.csv for all 139# metrics. 140assoc-metric-status() { 141 local job_dir=${1:-~/rappor/chrome-assoc-smoke/smoke3} 142 local out_dir=$job_dir/cooked 143 144 TOOLS-metric-status assoc $job_dir/assoc-task-status.csv $out_dir 145} 146 147"$@" 148