1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
|
#!/bin/bash
#
# End-to-end tests for the dashboard.
#
# Usage:
# ./regtest.sh <function name>
#
# NOTE: Must be run in this directory (rappor/pipeline).
set -o nounset
set -o pipefail
set -o errexit
# Create schema and params.
create-metadata() {
mkdir -p _tmp/metadata
echo 'Hello from regtest.sh'
local params_path=_tmp/metadata/regtest_params.csv
# Relying on $RAPPOR_SRC/regtest.sh
cp --verbose ../_tmp/python/demo1/case_params.csv $params_path
# For now, use the same map everywhere.
cat >_tmp/metadata/dist-analysis.csv <<EOF
var,map_filename
unif,map.csv
gauss,map.csv
exp,map.csv
m.domain,domain_map.csv
EOF
# Both single dimensional and multi dimensional metrics.
cat >_tmp/metadata/rappor-vars.csv <<EOF
metric,var,var_type,params
m,domain,string,m_params
m,flag..HTTPS,boolean,m_params
unif,,string,regtest_params
gauss,,string,regtest_params
exp,,string,regtest_params
EOF
}
# Create map files.
create-maps() {
mkdir -p _tmp/maps
# Use the same map for everyone now?
local map_path=_tmp/maps/map.csv
# Relying on $RAPPOR_SRC/regtest.sh
cp --verbose ../_tmp/python/demo1/case_map.csv $map_path
}
# Simulate different metrics.
create-counts() {
mkdir -p _tmp/counts
for date in 2015-12-01 2015-12-02 2015-12-03; do
mkdir -p _tmp/counts/$date
# TODO: Change params for each day.
cp --verbose \
../_tmp/python/demo1/1/case_counts.csv _tmp/counts/$date/unif_counts.csv
cp --verbose \
../_tmp/python/demo2/1/case_counts.csv _tmp/counts/$date/gauss_counts.csv
cp --verbose \
../_tmp/python/demo3/1/case_counts.csv _tmp/counts/$date/exp_counts.csv
done
}
dist-task-spec() {
local job_dir=$1
./task_spec.py dist \
--map-dir _tmp/maps \
--config-dir _tmp/metadata \
--output-base-dir $job_dir/raw \
--bad-report-out _tmp/bad_counts.csv \
"$@"
}
dist-job() {
local job_id=$1
local pat=$2
local job_dir=_tmp/$job_id
mkdir -p $job_dir/raw
local spec_list=$job_dir/spec-list.txt
find _tmp/counts/$pat -name \*_counts.csv \
| dist-task-spec $job_dir \
| tee $spec_list
./dist.sh decode-dist-many $job_dir $spec_list
./dist.sh combine-and-render-html _tmp $job_dir
}
dist() {
create-metadata
create-maps
create-counts
dist-job smoke1 '2015-12-01' # one day
dist-job smoke2 '2015-12-0[23]' # two days
}
# Simulate different metrics.
create-reports() {
mkdir -p _tmp/reports
for date in 2015-12-01 2015-12-02 2015-12-03; do
mkdir -p _tmp/reports/$date
# TODO: Change params for each day.
cp --verbose \
../bin/_tmp/reports.csv _tmp/reports/$date/m_reports.csv
done
}
assoc-task-spec() {
local job_dir=$1
./task_spec.py assoc \
--map-dir _tmp/maps \
--config-dir _tmp/metadata \
--output-base-dir $job_dir/raw \
"$@"
}
assoc-job() {
local job_id=$1
local pat=$2
local job_dir=_tmp/$job_id
mkdir -p $job_dir/raw $job_dir/config
local spec_list=$job_dir/spec-list.txt
find _tmp/reports/$pat -name \*_reports.csv \
| assoc-task-spec $job_dir \
| tee $spec_list
# decode-many calls decode_assoc.R, which expects this schema in the 'config'
# dir now. TODO: adjust this.
cp --verbose _tmp/metadata/rappor-vars.csv $job_dir/config
cp --verbose ../bin/_tmp/m_params.csv $job_dir/config
./assoc.sh decode-many $job_dir $spec_list
./assoc.sh combine-and-render-html _tmp $job_dir
}
# Copy some from bin/test.sh? The input _reports.csv files should be taken
# from there.
assoc() {
create-reports
cp --verbose ../bin/_tmp/domain_map.csv _tmp/maps
assoc-job smoke1-assoc '2015-12-01' # one day
assoc-job smoke2-assoc '2015-12-0[23]' # two days
}
"$@"
|
