ILIAS  release_5-3 Revision v5.3.23-19-g915713cf615
Aggregator.php
Go to the documentation of this file.
1<?php
2/*
3 * @author Andreas Åkre Solberg <andreas.solberg@uninett.no>
4 * @package SimpleSAMLphp
5 */
7{
8 private $statconfig;
9 private $statdir;
10 private $inputfile;
11 private $statrules;
12 private $offset;
13 private $metadata;
14 private $fromcmdline;
15 private $starttime;
16
20 public function __construct($fromcmdline = false)
21 {
22 $this->fromcmdline = $fromcmdline;
23 $this->statconfig = SimpleSAML_Configuration::getConfig('module_statistics.php');
24
25 $this->statdir = $this->statconfig->getValue('statdir');
26 $this->inputfile = $this->statconfig->getValue('inputfile');
27 $this->statrules = $this->statconfig->getValue('statrules');
28 $this->timeres = $this->statconfig->getValue('timeres');
29 $this->offset = $this->statconfig->getValue('offset', 0);
30 $this->metadata = null;
31
32 $this->starttime = time();
33 }
34
35 public function dumpConfig()
36 {
37 echo 'Statistics directory : ' . $this->statdir . "\n";
38 echo 'Input file : ' . $this->inputfile . "\n";
39 echo 'Offset : ' . $this->offset . "\n";
40 }
41
42 public function debugInfo()
43 {
44 echo 'Memory usage : ' . number_format(memory_get_usage() / (1024*1024), 2) . " MB\n";
45 }
46
47 public function loadMetadata()
48 {
49 $filename = $this->statdir . '/.stat.metadata';
50 $metadata = null;
51 if (file_exists($filename)) {
52 $metadata = unserialize(file_get_contents($filename));
53 }
54 $this->metadata = $metadata;
55 }
56
57 public function getMetadata()
58 {
59 return $this->metadata;
60 }
61
62 public function saveMetadata()
63 {
64 $this->metadata['time'] = time() - $this->starttime;
65 $this->metadata['memory'] = memory_get_usage();
66 $this->metadata['lastrun'] = time();
67
68 $filename = $this->statdir . '/.stat.metadata';
69 file_put_contents($filename, serialize($this->metadata), LOCK_EX);
70 }
71
72 public function aggregate($debug = false) {
73 $this->loadMetadata();
74
75 if (!is_dir($this->statdir)) {
76 throw new Exception('Statistics module: output dir do not exists [' . $this->statdir . ']');
77 }
78
79 if (!file_exists($this->inputfile)) {
80 throw new Exception('Statistics module: input file do not exists [' . $this->inputfile . ']');
81 }
82
83 $file = fopen($this->inputfile, 'r');
84
85 if ($file === false) {
86 throw new Exception('Statistics module: unable to open file [' . $this->inputfile . ']');
87 }
88
89 $logparser = new sspmod_statistics_LogParser(
90 $this->statconfig->getValue('datestart', 0), $this->statconfig->getValue('datelength', 15), $this->statconfig->getValue('offsetspan', 44)
91 );
92 $datehandler = array(
93 'default' => new sspmod_statistics_DateHandler($this->offset),
94 'month' => new sspmod_statistics_DateHandlerMonth($this->offset),
95 );
96
97 $notBefore = 0;
98 $lastRead = 0;
99 $lastlinehash = '-';
100
101 if (isset($this->metadata)) {
102 $notBefore = $this->metadata['notBefore'];
103 $lastlinehash = $this->metadata['lastlinehash'];
104 }
105
106 $lastlogline = 'sdfsdf';
107 $lastlineflip = false;
108 $results = array();
109
110 $i = 0;
111 // Parse through log file, line by line
112 while (!feof($file)) {
113 $logline = fgets($file, 4096);
114
115 // Continue if STAT is not found on line
116 if (!preg_match('/STAT/', $logline)) {
117 continue;
118 }
119
120 $i++;
121 $lastlogline = $logline;
122
123 // Parse log, and extract epoch time and rest of content.
124 $epoch = $logparser->parseEpoch($logline);
125 $content = $logparser->parseContent($logline);
126 $action = trim($content[5]);
127
128 if ($this->fromcmdline && ($i % 10000) == 0) {
129 echo("Read line " . $i . "\n");
130 }
131
132 if ($debug) {
133 echo("----------------------------------------\n");
134 echo('Log line: ' . $logline . "\n");
135 echo('Date parse [' . substr($logline, 0, $this->statconfig->getValue('datelength', 15)) . '] to [' . date(DATE_RFC822, $epoch) . ']' . "\n");
136 echo htmlentities(print_r($content, true));
137 if ($i >= 13) {
138 exit;
139 }
140 }
141
142 if ($epoch > $lastRead) {
143 $lastRead = $epoch;
144 }
145
146 if ($epoch === $notBefore) {
147 if (!$lastlineflip) {
148 if (sha1($logline) === $lastlinehash) {
149 $lastlineflip = true;
150 }
151 continue;
152 }
153 }
154
155 if ($epoch < $notBefore) {
156 continue;
157 }
158
159 // Iterate all the statrules from config.
160 foreach ($this->statrules as $rulename => $rule) {
161 $type = 'aggregate';
162
163 if (array_key_exists('type', $rule)) {
164 $type = $rule['type'];
165 }
166
167 if ($type !== 'aggregate') {
168 continue;
169 }
170
171 foreach ($this->timeres AS $tres => $tresconfig ) {
172 $dh = 'default';
173 if (isset($tresconfig['customDateHandler'])) {
174 $dh = $tresconfig['customDateHandler'];
175 }
176
177 $timeslot = $datehandler['default']->toSlot($epoch, $tresconfig['slot']);
178 $fileslot = $datehandler[$dh]->toSlot($epoch, $tresconfig['fileslot']);
179
180 if (isset($rule['action']) && ($action !== $rule['action'])) {
181 continue;
182 }
183
184 $difcol = self::getDifCol($content, $rule['col']);
185
186 if (!isset($results[$rulename][$tres][$fileslot][$timeslot]['_'])) {
187 $results[$rulename][$tres][$fileslot][$timeslot]['_'] = 0;
188 }
189 if (!isset($results[$rulename][$tres][$fileslot][$timeslot][$difcol])) {
190 $results[$rulename][$tres][$fileslot][$timeslot][$difcol] = 0;
191 }
192
193 $results[$rulename][$tres][$fileslot][$timeslot]['_']++;
194 $results[$rulename][$tres][$fileslot][$timeslot][$difcol]++;
195 }
196 }
197 }
198 $this->metadata['notBefore'] = $lastRead;
199 $this->metadata['lastline'] = $lastlogline;
200 $this->metadata['lastlinehash'] = sha1($lastlogline);
201 return $results;
202 }
203
204 private static function getDifCol($content, $colrule)
205 {
206 if (is_int($colrule)) {
207 return trim($content[$colrule]);
208 } elseif (is_array($colrule)) {
209 $difcols = array();
210 foreach ($colrule as $cr) {
211 $difcols[] = trim($content[$cr]);
212 }
213 return join('|', $difcols);
214 } else {
215 return 'NA';
216 }
217 }
218
219 private function cummulateData($previous, $newdata)
220 {
221 $dataset = array();
222 foreach ($previous as $slot => $dataarray) {
223 if (!array_key_exists($slot, $dataset)) {
224 $dataset[$slot] = array();
225 }
226 foreach ($dataarray as $key => $data) {
227 if (!array_key_exists($key, $dataset[$slot])) {
228 $dataset[$slot][$key] = 0;
229 }
230 $dataset[$slot][$key] += $data;
231 }
232 }
233 foreach ($newdata as $slot => $dataarray) {
234 if (!array_key_exists($slot, $dataset)) {
235 $dataset[$slot] = array();
236 }
237 foreach ($dataarray as $key => $data) {
238 if (!array_key_exists($key, $dataset[$slot])) {
239 $dataset[$slot][$key] = 0;
240 }
241 $dataset[$slot][$key] += $data;
242 }
243 }
244 return $dataset;
245 }
246
247 public function store($results)
248 {
249 $datehandler = array(
250 'default' => new sspmod_statistics_DateHandler($this->offset),
251 'month' => new sspmod_statistics_DateHandlerMonth($this->offset),
252 );
253
254 // Iterate the first level of results, which is per rule, as defined in the config.
255 foreach ($results as $rulename => $timeresdata) {
256 // Iterate over time resolutions
257 foreach ($timeresdata as $tres => $resres) {
258 $dh = 'default';
259 if (isset($this->timeres[$tres]['customDateHandler'])) {
260 $dh = $this->timeres[$tres]['customDateHandler'];
261 }
262
263 $filenos = array_keys($resres);
264 $lastfile = $filenos[count($filenos) - 1];
265
266 // Iterate the second level of results, which is the fileslot.
267 foreach ($resres as $fileno => $fileres) {
268 // Slots that have data.
269 $slotlist = array_keys($fileres);
270
271 // The last slot.
272 $maxslot = $slotlist[count($slotlist) - 1];
273
274 // Get start and end slot number within the file, based on the fileslot.
275 $start = (int)$datehandler['default']->toSlot(
276 $datehandler[$dh]->fromSlot($fileno, $this->timeres[$tres]['fileslot']),
277 $this->timeres[$tres]['slot']
278 );
279 $end = (int)$datehandler['default']->toSlot(
280 $datehandler[$dh]->fromSlot($fileno+1, $this->timeres[$tres]['fileslot']),
281 $this->timeres[$tres]['slot']
282 );
283
284 // Fill in missing entries and sort file results
285 $filledresult = array();
286 for ($slot = $start; $slot < $end; $slot++) {
287 if (array_key_exists($slot, $fileres)) {
288 $filledresult[$slot] = $fileres[$slot];
289 } else {
290 if ($lastfile == $fileno && $slot > $maxslot) {
291 $filledresult[$slot] = array('_' => null);
292 } else {
293 $filledresult[$slot] = array('_' => 0);
294 }
295 }
296 }
297
298 $filename = $this->statdir . '/' . $rulename . '-' . $tres . '-' . $fileno . '.stat';
299 if (file_exists($filename)) {
300 $previousData = unserialize(file_get_contents($filename));
301 $filledresult = $this->cummulateData($previousData, $filledresult);
302 }
303
304 // store file
305 file_put_contents($filename, serialize($filledresult), LOCK_EX);
306 }
307 }
308 }
309 $this->saveMetadata();
310 }
311}
memory_get_usage(true)/1024/1024)
date( 'd-M-Y', $objPHPExcel->getProperties() ->getCreated())
An exception for terminatinating execution or to throw for unit testing.
static getConfig($filename='config.php', $configSet='simplesaml')
Load a configuration file from a configuration set.
cummulateData($previous, $newdata)
Definition: Aggregator.php:219
static getDifCol($content, $colrule)
Definition: Aggregator.php:204
__construct($fromcmdline=false)
Constructor.
Definition: Aggregator.php:20
$action
$key
Definition: croninfo.php:18
$i
Definition: disco.tpl.php:19
$end
Definition: saml1-acs.php:18
$debug
Definition: loganalyzer.php:16
$type
if(!file_exists("$old.txt")) if( $old===$new) if(file_exists("$new.txt")) $file
$dataset
Definition: showstats.php:45
$rule
Definition: showstats.php:43
$fileslot
Definition: showstats.php:51
$results
Definition: svg-scanner.php:47