MetaGerSearch.php 13.1 KB
Newer Older
Dominik Hebeler's avatar
Dominik Hebeler committed
1
2
3
4
<?php

namespace App\Http\Controllers;

5
use App;
6
use App\MetaGer;
7
use Cache;
8
use Illuminate\Http\Request;
9
use LaravelLocalization;
10
use Log;
11
use View;
Dominik Hebeler's avatar
Dominik Hebeler committed
12
13
14

class MetaGerSearch extends Controller
{
Davide Aprea's avatar
Davide Aprea committed
15
    public function search(Request $request, MetaGer $metager, $timing = false)
16
    {
17
18
19
        if ($request->filled("chrome-plugin")) {
            return redirect(LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), "/plugin"));
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
20
21
22
23
        $timings = null;
        if ($timing) {
            $timings = ['starttime' => microtime(true)];
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
24
        $time = microtime(true);
25

26
        $focus = $request->input("focus", "web");
27

28
29
30
31
32
        if ($focus === "maps") {
            $searchinput = $request->input('eingabe', '');
            return redirect()->to('https://maps.metager.de/map/' . $searchinput . '/1240908.5493525574,6638783.2192695495,6');
        }

33
34
35
36
37
38
        # If there is no query parameter we redirect to the startpage
        $eingabe = $request->input('eingabe', '');
        if (empty(trim($eingabe))) {
            return redirect(LaravelLocalization::getLocalizedURL(LaravelLocalization::getCurrentLocale(), '/'));
        }

39
40
        # Mit gelieferte Formulardaten parsen und abspeichern:
        $metager->parseFormData($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
41
42
43
        if (!empty($timings)) {
            $timings["parseFormData"] = microtime(true) - $time;
        }
44

45
46
        # Nach Spezialsuchen überprüfen:
        $metager->checkSpecialSearches($request);
Dominik Hebeler's avatar
Dominik Hebeler committed
47
48
49
        if (!empty($timings)) {
            $timings["checkSpecialSearches"] = microtime(true) - $time;
        }
50

Dominik Hebeler's avatar
Dominik Hebeler committed
51
52
        # Search query can be empty after parsing the formdata
        # we will cancel the search in that case and show an error to the user
53
        if (empty($metager->getQ())) {
54
55
            return $metager->createView();
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
56

57
58
59
60
61
        $quicktips = $metager->createQuicktips();
        if (!empty($timings)) {
            $timings["createQuicktips"] = microtime(true) - $time;
        }

62
        # Suche für alle zu verwendenden Suchmaschinen als Job erstellen,
63
        # auf Ergebnisse warten und die Ergebnisse laden
Dominik Hebeler's avatar
Dominik Hebeler committed
64
        $metager->createSearchEngines($request, $timings);
65

66
67
        $metager->startSearch($timings);

68
        # Versuchen die Ergebnisse der Quicktips zu laden
69
        if ($quicktips !== null) {
Davide Aprea's avatar
Davide Aprea committed
70
71
72
73
74
75
            $quicktipResults = $quicktips->loadResults();
            if (!empty($timings)) {
                $timings["Loaded Quicktip Results"] = microtime(true) - $time;
            }
        } else {
            $quicktipResults = [];
76
77
        }

78
        $metager->waitForMainResults();
Dominik Hebeler's avatar
Dominik Hebeler committed
79
80
81
        if (!empty($timings)) {
            $timings["waitForMainResults"] = microtime(true) - $time;
        }
82
83

        $metager->retrieveResults();
Dominik Hebeler's avatar
Dominik Hebeler committed
84
85
86
        if (!empty($timings)) {
            $timings["retrieveResults"] = microtime(true) - $time;
        }
87

88
89
        # Alle Ergebnisse vor der Zusammenführung ranken:
        $metager->rankAll();
Dominik Hebeler's avatar
Dominik Hebeler committed
90
91
92
        if (!empty($timings)) {
            $timings["rankAll"] = microtime(true) - $time;
        }
93

94
        # Ergebnisse der Suchmaschinen kombinieren:
95
        $metager->prepareResults($timings);
96
        $admitad = [];
97
        $adgoal = [];
98
        if (!$metager->isApiAuthorized() && !$metager->isDummy()) {
Dominik Hebeler's avatar
Dominik Hebeler committed
99
            $newAdmitad = new \App\Models\Admitad($metager);
100
            if (!empty($newAdmitad->hash)) {
101
                $admitad[] = $newAdmitad;
Dominik Hebeler's avatar
Dominik Hebeler committed
102
            }
103
            $newAdgoal = new \App\Models\Adgoal($metager);
104
            if (!empty($newAdgoal->hash)) {
105
106
                $adgoal[] = $newAdgoal;
            }
107
        }
108

Dominik Hebeler's avatar
Dominik Hebeler committed
109
        $metager->parseAffiliates($admitad);
110
        $metager->parseAffiliates($adgoal);
111

112
113
114
115
116
        // Add Advertisement for Donations
        if (!$metager->isApiAuthorized() && !$metager->isDummy()) {
            $metager->addDonationAdvertisement();
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
117
118
119
120
121
122
123
        $finished = true;
        foreach ($metager->getEngines() as $engine) {
            if ($engine->loaded) {
                $engine->setNew(false);
                $engine->markNew();
            }
        }
Davide Aprea's avatar
Davide Aprea committed
124
125
126
127
128
        try {
            Cache::put("loader_" . $metager->getSearchUid(), [
                "metager" => [
                    "apiAuthorized" => $metager->isApiAuthorized(),
                ],
129
                "admitad" => $admitad,
130
                "adgoal" => $adgoal,
Davide Aprea's avatar
Davide Aprea committed
131
132
133
134
                "engines" => $metager->getEngines(),
            ], 60 * 60);
        } catch (\Exception $e) {
            Log::error($e->getMessage());
135
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
136
137
138
        if (!empty($timings)) {
            $timings["Filled resultloader Cache"] = microtime(true) - $time;
        }
139

140
        # Die Ausgabe erstellen:
141
        $resultpage = $metager->createView($quicktipResults);
Dominik Hebeler's avatar
Dominik Hebeler committed
142

Dominik Hebeler's avatar
Dominik Hebeler committed
143
144
145
146
147
148
149
150
        if (!empty($timings)) {
            $timings["createView"] = microtime(true) - $time;
        }

        if ($timings) {
            dd($timings);
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
151
        $registry = \Prometheus\CollectorRegistry::getDefault();
152
153
        $counter = $registry->getOrRegisterCounter('metager', 'result_counter', 'counts total number of returned results', []);
        $counter->incBy(sizeof($metager->getResults()));
Dominik Hebeler's avatar
Dominik Hebeler committed
154
155
        $counter = $registry->getOrRegisterCounter('metager', 'query_counter', 'counts total number of search queries', []);
        $counter->inc();
156

157
158
159
160
        // Splitting the response return into multiple parts.
        // This might speed up page view time for users with slow network
        $responseArray = str_split($resultpage->render(), 1024);
        foreach ($responseArray as $responsePart) {
161
            echo ($responsePart);
162
163
            flush();
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
164
165
        $requestTime = microtime(true) - $time;
        \App\PrometheusExporter::Duration($requestTime, "request_time");
Dominik Hebeler's avatar
Dominik Hebeler committed
166
167
    }

Dominik Hebeler's avatar
Dominik Hebeler committed
168
169
170
171
172
173
174
175
    public function searchTimings(Request $request, MetaGer $metager)
    {
        $request->merge([
            'eingabe' => "Hannover",
        ]);
        return $this->search($request, $metager, true);
    }

176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
    public function loadMore(Request $request)
    {
        /**
         * There are three forms of requests to the resultpage
         * 1. Initial Request: Loads the fastest searchengines and sends them to the user
         * 2. Load more results (with JS): Loads new search engines that answered after the initial request was send
         * 3. Load more results (without JS): Loads new search engines that answered within 1s timeout
         */
        if ($request->filled('loadMore') && $request->filled('script') && $request->input('script') === "yes") {
            return $this->loadMoreJS($request);
        }
    }

    private function loadMoreJS(Request $request)
    {
Dominik Hebeler's avatar
Dominik Hebeler committed
191
        $request->request->add(["javascript" => true]);
192
193
194
        # Create a MetaGer Instance with the supplied hash
        $hash = $request->input('loadMore', '');

195
196
197
198
199
200
        # Parser Skripte einhängen
        $dir = app_path() . "/Models/parserSkripte/";
        foreach (scandir($dir) as $filename) {
            $path = $dir . $filename;
            if (is_file($path)) {
                require_once $path;
201
            }
202
        }
203

204
205
        $cached = Cache::get($hash);
        if ($cached === null) {
206
207
            return response()->json(['finished' => true]);
        }
208

209
210
        $engines = $cached["engines"];
        $adgoal = $cached["adgoal"];
211
        $admitad = $cached["admitad"];
212
213
        $mg = $cached["metager"];

Dominik Hebeler's avatar
Dominik Hebeler committed
214
        $metager = new MetaGer(substr($hash, strpos($hash, "loader_") + 7));
215
        $metager->setApiAuthorized($mg["apiAuthorized"]);
216

217
        $metager->parseFormData($request, false);
218
        $metager->setJsEnabled(true);
219
220
221
222
        # Nach Spezialsuchen überprüfen:
        $metager->checkSpecialSearches($request);
        $metager->restoreEngines($engines);

Dominik Hebeler's avatar
Dominik Hebeler committed
223
224
225
        # Checks Cache for engine Results
        $metager->checkCache();

226
        $metager->retrieveResults();
227

228
229
        $metager->rankAll();
        $metager->prepareResults();
230

231
        if (!$metager->isApiAuthorized() && !$metager->isDummy()) {
Dominik Hebeler's avatar
Dominik Hebeler committed
232
            $newAdmitad = new \App\Models\Admitad($metager);
233
            if (!empty($newAdmitad->hash)) {
234
                $admitad[] = $newAdmitad;
Dominik Hebeler's avatar
Dominik Hebeler committed
235
            }
236
            $newAdgoal = new \App\Models\Adgoal($metager);
237
            if (!empty($newAdgoal->hash)) {
238
239
                $adgoal[] = $newAdgoal;
            }
240
241
        }

Dominik Hebeler's avatar
Dominik Hebeler committed
242
        $admitadFinished = $metager->parseAffiliates($admitad);
243
        $adgoalFinished = $metager->parseAffiliates($adgoal);
244
245
246
247

        $result = [
            'finished' => true,
            'newResults' => [],
248
            'changedResults' => [],
249
250
251
        ];
        $result["nextSearchLink"] = $metager->nextSearchLink();

252
        $newResults = 0;
253
        foreach ($metager->getResults() as $index => $resultTmp) {
Davide Aprea's avatar
Davide Aprea committed
254
            if ($resultTmp->new || $resultTmp->changed) {
255
256
257
                if ($metager->getFokus() !== "bilder") {
                    $view = View::make('layouts.result', ['index' => $index, 'result' => $resultTmp, 'metager' => $metager]);
                    $html = $view->render();
Davide Aprea's avatar
Davide Aprea committed
258
                    if (!$resultTmp->new && $resultTmp->changed) {
259
260
261
262
                        $result['changedResults'][$index] = $html;
                    } else {
                        $result['newResults'][$index] = $html;
                    }
263
264
265
266
                    $result["imagesearch"] = false;
                } else {
                    $view = View::make('layouts.image_result', ['index' => $index, 'result' => $resultTmp, 'metager' => $metager]);
                    $html = $view->render();
Davide Aprea's avatar
Davide Aprea committed
267
                    if (!$resultTmp->new && $resultTmp->changed) {
268
269
270
271
                        $result['changedResults'][$index] = $html;
                    } else {
                        $result['newResults'][$index] = $html;
                    }
272
                    $result["imagesearch"] = true;
273
                }
274
                $newResults++;
275
276
            }
        }
277

Dominik Hebeler's avatar
Dominik Hebeler committed
278
        $finished = true;
Dominik Hebeler's avatar
Dominik Hebeler committed
279
        $enginesLoaded = [];
Dominik Hebeler's avatar
Dominik Hebeler committed
280
281
        foreach ($engines as $engine) {
            if (!$engine->loaded) {
Dominik Hebeler's avatar
Dominik Hebeler committed
282
                $enginesLoaded[$engine->name] = false;
Dominik Hebeler's avatar
Dominik Hebeler committed
283
284
                $finished = false;
            } else {
Dominik Hebeler's avatar
Dominik Hebeler committed
285
                $enginesLoaded[$engine->name] = true;
Dominik Hebeler's avatar
Dominik Hebeler committed
286
287
288
289
                $engine->setNew(false);
                $engine->markNew();
            }
        }
290

Dominik Hebeler's avatar
Dominik Hebeler committed
291
        if (!$adgoalFinished || !$admitadFinished) {
292
            $finished = false;
293
        }
Dominik Hebeler's avatar
Dominik Hebeler committed
294
295

        $result["finished"] = $finished;
Dominik Hebeler's avatar
Dominik Hebeler committed
296
        $result["engines"] = $enginesLoaded;
Dominik Hebeler's avatar
Dominik Hebeler committed
297

298
        if ($newResults > 0) {
299
300
301
302
            $registry = \Prometheus\CollectorRegistry::getDefault();
            $counter = $registry->getOrRegisterCounter('metager', 'result_counter', 'counts total number of returned results', []);
            $counter->incBy($newResults);
        }
303
        // Update new Engines
304
305
306
307
        Cache::put("loader_" . $metager->getSearchUid(), [
            "metager" => [
                "apiAuthorized" => $metager->isApiAuthorized(),
            ],
308
            "admitad" => $admitad,
309
            "adgoal" => $adgoal,
310
311
            "engines" => $metager->getEngines(),
        ], 1 * 60);
312
313
314
315

        # JSON encoding will fail if invalid UTF-8 Characters are in this string
        # mb_convert_encoding will remove thise invalid characters for us
        $result = mb_convert_encoding($result, "UTF-8", "UTF-8");
316
317
318
        return response()->json($result);
    }

319
320
321
322
323
324
325
326
    public function botProtection($redirect)
    {
        $hash = md5(date('YmdHi'));
        return view('botProtection')
            ->with('hash', $hash)
            ->with('r', $redirect);
    }

327
328
    public function get($url)
    {
329
        $ctx = stream_context_create(array('http' => array('timeout' => 2)));
330
        return file_get_contents($url, false, $ctx);
331
    }
332
333
334
335
336
337

    private function startsWith($haystack, $needle)
    {
        $length = strlen($needle);
        return (substr($haystack, 0, $length) === $needle);
    }
338
339
340

    public function tips(Request $request)
    {
341
        $tipserver = '';
342
        if (\App::environment() === "development") {
343
344
345
346
            $tipserver = "https://dev.quicktips.metager.de/1.1/tips.xml";
        } else {
            $tipserver = "https://quicktips.metager.de/1.1/tips.xml";
        }
347
348
349
        if (LaravelLocalization::getCurrentLocale() == "en") {
            $tipserver .= "?locale=en";
        }
350
        $tips_text = file_get_contents($tipserver);
351
        $tips = [];
352
        try {
Dominik Hebeler's avatar
Dominik Hebeler committed
353
            $tips_xml = \simplexml_load_string($tips_text);
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368

            $tips_xml->registerXPathNamespace('mg', 'http://metager.de/tips/');
            $tips_xml = $tips_xml->xpath('mg:tip');
            foreach ($tips_xml as $tip_xml) {
                $tips[] = $tip_xml->__toString();
            }
        } catch (\Exception $e) {
            Log::error("A problem occurred loading tips from the tip server.");
            Log::error($e->getMessage());
            abort(500);
        }
        return view('tips')
            ->with('title', trans('tips.title'))
            ->with('tips', $tips);
    }
369

370
    public function quicktips(Request $request)
371
    {
372
        $search = $request->input('search', '');
373
        $quotes = $request->input('quotes', 'on');
374
        if (empty($search)) {
Dominik Hebeler's avatar
Dominik Hebeler committed
375
            abort(404);
376
377
        }

378
        $quicktips = new \App\Models\Quicktips\Quicktips($search, $quotes);
Dominik Hebeler's avatar
Dominik Hebeler committed
379
380
381
        return view('quicktips')
            ->with('quicktips', $quicktips->getResults())
            ->with('search', $search);
382
    }
383
}