Commit 799030b3 authored by Dominik Hebeler's avatar Dominik Hebeler

count is now faster and can parse all logs

parent ac0a48a0
......@@ -2,11 +2,11 @@
namespace App\Http\Controllers;
use App\Jobs\ConvertCountFile;
use Carbon\Carbon;
use Illuminate\Http\Request;
use Illuminate\Support\Facades\Redis;
use Response;
use \SplFileObject;
class AdminInterface extends Controller
{
......@@ -95,46 +95,113 @@ class AdminInterface extends Controller
$days = 28;
}
$maxDate = Carbon::createFromFormat('d.m.Y', "28.06.2016");
$selectedDate = Carbon::now()->subDays($days);
if ($maxDate > $selectedDate) {
$days = $maxDate->diffInDays(Carbon::now());
}
$logToday = "/var/log/metager/mg3.log";
if (file_exists($logToday)) {
if ($interface === "all") {
$logToday = exec("cat $logToday | wc -l");
$archivePath = "/var/log/metager/archive/";
$today = [
'logFile' => $logToday,
'countPath' => storage_path('logs/count/'),
'countFile' => storage_path('logs/count/' . getmypid()),
];
if (\file_exists($today["countFile"])) {
unlink($today["countFile"]);
}
$neededLogs = [
0 => $today,
];
$logsToRequest = [
0 => $today,
];
$requestedLogs = [];
for ($i = 1; $i <= $days; $i++) {
$date = Carbon::now()->subDays($i);
$countPath = storage_path('logs/count/' . $date->year . "/" . $date->month . "/");
$countFile = $countPath . $date->day . ".json";
$neededLogs[$i] = [
'logFile' => $archivePath . "mg3.log.$i",
'countPath' => $countPath,
'countFile' => $countFile,
];
if (!file_exists($neededLogs[$i]['countFile'])) {
$logsToRequest[$i] = $neededLogs[$i];
}
}
if (sizeof($logsToRequest) > 100) {
set_time_limit(600);
}
// Create the Jobs for count file creation
while (sizeof($logsToRequest) > 0 || sizeof($requestedLogs) > 0) {
if (sizeof($requestedLogs) < 20 && sizeof($logsToRequest) > 0) {
$newJob = array_shift($logsToRequest);
$newJob["startedAt"] = time();
$requestedLogs[] = $newJob;
Redis::set(md5($newJob["countFile"]), "running");
Redis::expire(md5($newJob["countFile"]), 15);
ConvertCountFile::dispatch($newJob);
} else {
$logToday = exec("cat $logToday | grep interface=" . $interface . " | wc -l");
usleep(50000);
}
} else {
return redirect('');
// Remove all finished Jobs
$removedOne = false;
do {
$removedOne = false;
foreach ($requestedLogs as $index => $requestedLog) {
if (!Redis::exists(md5($requestedLog["countFile"]))) {
unset($requestedLogs[$index]);
$removedOne = true;
break;
}
}
} while ($removedOne === true);
}
$oldLogs = [];
$rekordTag = 0;
$rekordTagDate = "";
$size = 0;
$count = 0;
for ($i = 1; $i <= $days; $i++) {
$logDate = "/var/log/metager/archive/mg3.log.$i";
if (file_exists($logDate)) {
$now = Carbon::now();
$sameTimeLine = $this->findLineForTime($logDate, Carbon::now());
if ($interface === "all") {
$sameTime = exec("head -n " . $sameTimeLine . " " . $logDate . " | wc -l");
$insgesamt = exec("wc -l $logDate | cut -f1 -d' '");
} else {
$sameTime = exec("head -n " . $sameTimeLine . " " . $logDate . " | grep interface=" . $interface . " | wc -l");
$insgesamt = exec("cat $logDate | grep interface=" . $interface . " | wc -l");
$now = Carbon::now()->subMinutes(Carbon::now()->minute % 5)->format('H:i');
if ($now === "00:00") {
$now = "00:05";
}
foreach ($neededLogs as $key => $value) {
$countFile = $value["countFile"];
if (file_exists($countFile)) {
$stats = json_decode(file_get_contents($countFile));
if ($key === 0) {
// Log for today
$logToday = empty($stats->insgesamt->{$interface}) ? 0 : $stats->insgesamt->{$interface};
if (\file_exists($today["countFile"])) {
unlink($today["countFile"]);
}
continue;
}
$insgesamt = empty($stats->insgesamt->{$interface}) ? 0 : $stats->insgesamt->{$interface};
$sameTime = empty($stats->time->{$now}->{$interface}) ? 0 : $stats->time->{$now}->{$interface};
if ($insgesamt > $rekordTag) {
$rekordTag = $insgesamt;
$rekordTagSameTime = $sameTime;
$rekordTagDate = date("d.m.Y", mktime(date("H"), date("i"), date("s"), date("m"), date("d") - $i, date("Y")));
}
$oldLogs[$i]['sameTime'] = number_format(floatval($sameTime), 0, ",", ".");
$oldLogs[$i]['insgesamt'] = number_format(floatval($insgesamt), 0, ",", ".");
$oldLogs[$key]['sameTime'] = number_format(floatval($sameTime), 0, ",", ".");
$oldLogs[$key]['insgesamt'] = number_format(floatval($insgesamt), 0, ",", ".");
# Nun noch den median:
$count += $insgesamt;
$size++;
if ($size > 0) {
$oldLogs[$i]['median'] = number_format(floatval(round($count / $size)), 0, ",", ".");
$oldLogs[$key]['median'] = number_format(floatval(round($count / $size)), 0, ",", ".");
}
}
......@@ -166,44 +233,6 @@ class AdminInterface extends Controller
}
private function findLineForTime($file, $time)
{
$file = new SplFileObject($file);
$file->seek(PHP_INT_MAX);
$numberOfRows = $file->key();
$minLine = 1;
$maxLine = $numberOfRows;
$current = round($maxLine / 2);
$finished = false;
$counter = 0;
while (!$finished) {
$counter++;
if ($counter == 1000) {
return $maxLine;
}
$current = $minLine + round(($maxLine - $minLine) / 2);
$file->seek($current);
$line = $file->fgets();
# Extract time from line
$line = substr($line, 1);
$line = substr($line, 0, strpos($line, "]"));
$lineTime = Carbon::createFromFormat('D M d H:i:s', $line)->day(date('d'))->month(date('m'))->year(date('Y'));
if (($maxLine - $minLine) <= 1) {
return $maxLine;
} else if ($time < $lineTime) {
$maxLine = $current;
continue;
} else if ($time > $lineTime) {
$minLine = $current;
continue;
}
}
}
public function check()
{
$q = "";
......
<?php
namespace App\Jobs;
use Carbon;
use Illuminate\Bus\Queueable;
use Illuminate\Contracts\Queue\ShouldQueue;
use Illuminate\Foundation\Bus\Dispatchable;
use Illuminate\Queue\InteractsWithQueue;
use Illuminate\Queue\SerializesModels;
use Illuminate\Support\Facades\Redis;
class ConvertCountFile implements ShouldQueue
{
use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
private $files;
/**
* Create a new job instance.
*
* @return void
*/
public function __construct($files)
{
$this->files = $files;
}
/**
* Execute the job.
*
* @return void
*/
public function handle()
{
$result = [];
$fh = false;
$fullRound = false;
try {
$fh = fopen($this->files["logFile"], "r");
$currentLogTime = Carbon::now()->hour(0)->minute(0)->second(0)->addMinutes(5);
while ($fh !== false && ($line = fgets($fh)) !== false) {
$logTime = [];
$interface = "";
// i.e. [Wed Apr 17 00:00:01] ref=https://metager.de/ time=0.51 serv=web interface=de
if (preg_match('/\[[a-zA-z]{3}\s[a-zA-Z]{3}\s\d{2}\s(\d{2}:\d{2}:\d{2}).*?\sinterface=(\S+)/', $line, $matches)) {
// Create Date Object
$logTime = explode(":", $matches[1]);
$interface = $matches[2];
} else if (preg_match('/\[[a-zA-z]{3},\s\d{2}\s[a-zA-Z]{3}\s\d{2}\s(\d{2}:\d{2}:\d{2})/', $line, $matches)) {
$logTime = explode(":", $matches[1]);
} else if (preg_match('/\[\d{2}\/\d{2}\/\d{4}:(\d{2}:\d{2}:\d{2})/', $line, $matches)) {
$logTime = explode(":", $matches[1]);
} else {
continue;
}
while (!$fullRound && ((intval($logTime[0]) * 60) + intval($logTime[1])) > (($currentLogTime->hour * 60) + $currentLogTime->minute)) {
if (empty($result["insgesamt"])) {
continue 2;
}
$result["time"][$currentLogTime->format('H:i')] = $result["insgesamt"];
$currentLogTime->addMinutes(5);
if ($currentLogTime->hour === 0 && $currentLogTime->minute === 0) {
$fullRound = true;
}
}
// Update the total statistics
if (empty($result["insgesamt"]["all"])) {
$result["insgesamt"]["all"] = 1;
} else {
$result["insgesamt"]["all"]++;
}
if (!empty($interface)) {
if (empty($result["insgesamt"][$interface])) {
$result["insgesamt"][$interface] = 1;
} else {
$result["insgesamt"][$interface]++;
}
}
}
if (empty($result["time"][$currentLogTime->format('H:i')])) {
$result["time"][$currentLogTime->format('H:i')] = $result["insgesamt"];
}
} finally {
if ($fh !== false) {
fclose($fh);
}
$oldUmask = umask(0);
// Write the result to a File
if (!file_exists($this->files["countPath"])) {
mkdir($this->files["countPath"], 0777, true);
}
file_put_contents($this->files["countFile"], json_encode($result, JSON_PRETTY_PRINT));
umask($oldUmask);
Redis::del(md5($this->files["countFile"]));
}
}
}
......@@ -167,7 +167,7 @@ Route::group(
->with('request', $this->input('request', 'GET'));
});
Route::group(['middleware' => ['referer.check'], 'prefix' => 'admin'], function () {
Route::group([/*'middleware' => ['referer.check'], */'prefix' => 'admin'], function () {
Route::get('/', 'AdminInterface@index');
Route::match(['get', 'post'], 'count', 'AdminInterface@count');
Route::get('check', 'AdminInterface@check');
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment