Commit 4001e042 authored by Dominik Hebeler's avatar Dominik Hebeler

Merge branch '859-filter-options-for-metager' into 'development'

Resolve "Filter Options for MetaGer"

Closes #859, #793, #849, #846, #848, #844, #836, #841, #835, #830, #828, #838, #831, #834, #825, #820, #816, #813, #821, #823, #819, #818, #805, #809, #817, #810, #812, #814, #811, #707, #731, #699, #808, #754, #800, #778, #804, #799, #798, #801, #796, #795, #794, #788, #789, #767, #786, #782, #785, #784, #783, #781, #773, #776, #777, #770, #780, #772, #769, #774, #763, #759, #760, #761, #758, #746, #753, #756, #748, #629, #750, #751, #744, #747, #652, #676, #693, #742, #745, #584, #743, #739, #697, #732, #733, #734, #736, #737, #738, #729, #712, #704, #701, #710, #728, #727, #711, #703, #688, #638, #690, #691, #689, #646, #677, #684, #671, #670, #663, #659, #660, #658, #657, #654, #651, #653, #648, #647, #622, #595, #634, #632, #627, #618, #624, #621, #613, #608, #570, #603, #600, #588, #593, #594, #590, #591, #587, #216, #503, #574, #582, #581, #550, #359, #580, #579, #578, #576, #573, #575, #566, #568, #572, #564, #562, #561, #560, #530, #557, #556, #554, #558, #540, #555, #552, #553, #549, #551, #468, #546, #542, #545, #541, #538, #539, #537, #536, #535, #534, #525, #533, #531, #526, #504, #524, #519, #521, #517, #513, #408, #516, #515, #514, #512, #495, #499, #510, #509, #508, #507, #483, #493, #466, #494, #506, #505, #500, #487, #501, #496, #498, #485, #489, #488, #486, #481, #477, #470, #461, #480, #474, #449, #471, #237, #446, #462, #467, #465, #417, #460, #457, #456, #452, #454, #453, #450, #433, #448, #447, #361, #445, #441, #440, #438, #439, #432, #437, #435, #347, #425, #429, #428, #430, #363, #407, #406, #245, #157, #421, #405, #351, #414, #411, #410, #409, #398, #397, #403, #402, #399, #393, #396, #395, #394, #392, #391, #387, #386, #385, #318, #365, #384, #382, #381, #380, #352, #379, #377, #348, #350, #375, #376, #374, #373, #372, #311, #357, #362, #358, #370, #366, #356, #360, #354, #341, #319, #345, #343, #344, #310, #89, #342, #333, #264, #334, #330, #329, #328, #326, #321, #327, #324, #320, #317, #316, #313, #309, #306, #295, #305, #301, #303, #299, #296, #293, #292, #290, #285, #287, #272, #284, #280, #226, #278, #273, #276, #274, #233, #271, #269, #266, #261, #265, #263, #257, #235, #255, #229, #254, #252, #250, #249, #247, #248, #246, #244, #242, #243, #219, #238, #236, #234, #228, #222, #221, #211, #225, #187, #206, #87, #215, #214, #212, #210, #208, #207, #204, #203, #201, #202, #182, #200, #197, #199, #195, #194, #192, #193, #191, #190, #189, #188, #137, #162, #96, #160, #179, #177, #174, #175, #169, #173, #172, #170, #166, #168, #165, #164, #159, #155, #153, #97, #156, #134, #142, #124, #154, #135, #146, #151, #150, #149, #125, #92, #145, #140, #98, #133, #105, #90, #138, #126, #128, #132, #129, #123, #121, #37, #108, #119, #117, #116, #113, #112, #107, #109, #106, #50, #99, #103, #102, #72, #74, #38, #94, #76, #75, #41, #40, #73, #71, #70, #66, #57, #67, #65, #64, #63, #55, #56, #59, #61, and #35

See merge request !1365
parents 2ee13bc2 b9683f8e
......@@ -18,14 +18,14 @@ update(144.76.113.134):
- cd MetaGer_neu
- composer install
- scp -P 63824 metager@metager3.de:~/.env .
- scp -P 63824 metager@metager3.de:~/sumas.xml config/
- scp -P 63824 metager@metager3.de:~/sumasEn.xml config/
- scp -P 63824 metager@metager3.de:~/sumas.json config/
- scp -P 63824 metager@metager3.de:~/sumasEn.json config/
- scp -P 63824 metager@metager3.de:~/blacklistUrl.txt config/
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log
- touch storage/logs/worker.log
- touch database/metager.sqlite
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod 777 config/sumas.json config/sumas.json database/metager.sqlite
- chmod -R 777 storage
- chmod -R 777 bootstrap/cache
- npm install
......@@ -58,14 +58,14 @@ update(metager2):
- cd MetaGer_neu
- composer install
- scp -P 63824 metager@metager3.de:~/.env .
- scp -P 63824 metager@metager3.de:~/sumas.xml config/
- scp -P 63824 metager@metager3.de:~/sumasEn.xml config/
- scp -P 63824 metager@metager3.de:~/sumas.json config/
- scp -P 63824 metager@metager3.de:~/sumas.json config/
- scp -P 63824 metager@metager3.de:~/blacklistUrl.txt config/
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log
- touch storage/logs/worker.log
- touch database/metager.sqlite
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod 777 config/sumas.json config/sumas.json database/metager.sqlite
- chmod -R 777 storage
- chmod -R 777 bootstrap/cache
- npm install
......@@ -98,14 +98,14 @@ update(metager3.de):
- cd MetaGer_neu
- composer install
- scp -P 63824 metager@metager3.de:~/.env .
- scp -P 63824 metager@metager3.de:~/sumas.xml config/
- scp -P 63824 metager@metager3.de:~/sumasEn.xml config/
- scp -P 63824 metager@metager3.de:~/sumas.json config/
- scp -P 63824 metager@metager3.de:~/sumas.json config/
- scp -P 63824 metager@metager3.de:~/blacklistUrl.txt config/
- scp -P 63824 metager@metager3.de:~/blacklistDomains.txt config/
- touch storage/logs/laravel.log
- touch storage/logs/worker.log
- touch database/metager.sqlite
- chmod 777 config/sumas.xml config/sumasEn.xml database/metager.sqlite
- chmod 777 config/sumas.json config/sumas.json database/metager.sqlite
- chmod -R 777 storage
- chmod -R 777 bootstrap/cache
- npm install
......
......@@ -161,6 +161,7 @@ class Searcher implements ShouldQueue
// Set this URL to the Curl handle
curl_setopt($this->ch, CURLOPT_URL, $url);
$result = curl_exec($this->ch);
$this->connectionInfo = curl_getinfo($this->ch);
return $result;
}
......@@ -203,10 +204,13 @@ class Searcher implements ShouldQueue
}
if ($this->headers !== null) {
$headers = [];
foreach ($this->headers as $key => $value) {
$headers[] = $key . ":" . $value;
}
# Headers are in the Form:
# <key>:<value>;<key>:<value>
$headerArray = explode(";", $this->headers);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headerArray);
curl_setopt($ch, CURLOPT_HTTPHEADER, $headers);
}
return $ch;
......
This diff is collapsed.
......@@ -31,7 +31,6 @@ class Result
# Erstellt ein neues Ergebnis
public function __construct($provider, $titel, $link, $anzeigeLink, $descr, $gefVon, $gefVonLink, $sourceRank, $additionalInformation = [])
{
$provider = simplexml_load_string($provider);
$this->titel = strip_tags(trim($titel));
$this->link = trim($link);
$this->anzeigeLink = trim($anzeigeLink);
......@@ -52,8 +51,8 @@ class Result
$this->sourceRank = 20;
}
$this->sourceRank = 20 - $this->sourceRank;
if (isset($provider["engineBoost"])) {
$this->engineBoost = floatval($provider["engineBoost"]->__toString());
if (isset($provider->{"engine-boost"})) {
$this->engineBoost = floatval($provider->{"engine-boost"});
} else {
$this->engineBoost = 1;
}
......@@ -275,22 +274,6 @@ class Result
}
}
*/
/* Der Host-Filter der sicherstellt,
* dass von jedem Host maximal 3 Links angezeigt werden.
* Diese Überprüfung führen wir unter bestimmten Bedingungen nicht durch.
*/
if (($metager->getSite() === "" || $metager->getSite() === null) &&
strpos($this->strippedHost, "ncbi.nlm.nih.gov") === false &&
strpos($this->strippedHost, "twitter.com") === false &&
strpos($this->strippedHost, "www.ladenpreis.net") === false &&
strpos($this->strippedHost, "ncbi.nlm.nih.gov") === false &&
strpos($this->strippedHost, "www.onenewspage.com") === false &&
$this->gefVon !== "Shopzilla") {
$count = $metager->getHostCount($this->strippedHost);
if ($count >= 3) {
return false;
}
}
/* Der Dublettenfilter, der sicher stellt,
* dass wir nach Möglichkeit keinen Link doppelt in der Ergebnisliste haben.
......
This diff is collapsed.
......@@ -9,9 +9,9 @@ use Symfony\Component\DomCrawler\Crawler;
class Allesklar extends Searchengine
{
protected $tds = "";
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
public function __construct($name, \StdClass $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
parent::__construct($name, $engine, $metager);
}
public function loadResults($result)
......@@ -43,7 +43,7 @@ class Allesklar extends Searchengine
$link,
$link,
$descr,
$this->displayName,$this->homepage,
$this->engine->{"display-name"},$this->engine->homepage,
$this->counter
);
} catch (\Exception $e) {
......
......@@ -9,9 +9,9 @@ class BASE extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
public function __construct($name, \StdClass $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
parent::__construct($name, $engine, $metager);
}
public function loadResults($result)
......@@ -49,7 +49,7 @@ class BASE extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->engine->{"display-name"},$this->engine->homepage,
$this->counter
);
}
......
......@@ -8,9 +8,9 @@ class Beammachine extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
public function __construct($name, \StdClass $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
parent::__construct($name, $engine, $metager);
}
public function loadResults($result)
......@@ -34,7 +34,7 @@ class Beammachine extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->engine->{"display-name"},$this->engine->homepage,
$this->counter
);
}
......
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
use Symfony\Component\DomCrawler\Crawler;
use Log;
class Bing extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults($result)
{
try {
$crawler = new Crawler($result);
$crawler->filter('ol#b_results > li.b_algo')->each(function (Crawler $node, $i) {
$title = $node->filter('li h2 > a')->text();
$link = $node->filter('li h2 > a')->attr('href');
$anzeigeLink = $link;
$descr = $node->filter('li div > p')->text();
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->counter
);
});
} catch (\Exception $e) {
Log::error("A problem occurred parsing results from $this->name:");
Log::error($e->getMessage());
return;
}
}
}
......@@ -9,9 +9,9 @@ class BingBilder extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
public function __construct($name, \stdClass $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
parent::__construct($name, $engine, $metager);
}
public function loadResults($result)
......@@ -33,7 +33,7 @@ class BingBilder extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->displayName, $this->homepage,
$this->engine->{"display-name"}, $this->engine->homepage,
$this->counter,
['image' => $image]
);
......@@ -59,12 +59,9 @@ class BingBilder extends Searchengine
return;
}
$next = new BingBilder(simplexml_load_string($this->engine), $metager);
if (\str_contains($next->getString, "&offset=")) {
$next->getString = preg_replace("/&offset=.*/si", "", $next->getString);
}
$next->getString .= "&offset=" . $nextOffset;
$next->hash = md5($next->host . $next->getString . $next->port . $next->name);
$newEngine = unserialize(serialize($this->engine));
$newEngine->{"get-parameter"}->offset = $nextOffset;
$next = new BingBilder($this->name, $newEngine, $metager);
$this->next = $next;
} catch (\Exception $e) {
......
<?php
namespace app\Models\parserSkripte;
use App\Models\Searchengine;
use Symfony\Component\DomCrawler\Crawler;
class Bing_bilder extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
}
public function loadResults($result)
{
$crawler = new Crawler($result);
$crawler->filter('div#b_content div.item')->each(function (Crawler $node, $i) {
$title = $node->filter('div.meta > a.tit')->text();
$link = $node->filter('div.meta > a.tit')->attr("href");
$anzeigeLink = $link;
$descr = $node->filter('div.meta > div.des')->text();
$image = $node->filter('a.thumb img')->attr("src");
$this->counter++;
$this->results[] = new \App\Models\Result(
$this->engine,
$title,
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->counter,
['image' => $image]
);
});
}
}
......@@ -9,9 +9,9 @@ class Blogsearch extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
public function __construct($name, \StdClass $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
parent::__construct($name, $engine, $metager);
}
public function loadResults($result)
......@@ -36,7 +36,7 @@ class Blogsearch extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->engine->{"display-name"},$this->engine->homepage,
$this->counter
);
}
......
......@@ -9,9 +9,9 @@ class Dailymotion extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
public function __construct($name, \StdClass $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
parent::__construct($name, $engine, $metager);
}
public function loadResults($result)
......@@ -37,7 +37,7 @@ class Dailymotion extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->engine->{"display-name"},$this->engine->homepage,
$this->counter,
['partnershop' => false]
);
......
......@@ -9,9 +9,9 @@ class Dart extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
public function __construct($name, \StdClass $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
parent::__construct($name, $engine, $metager);
}
public function loadResults($result)
......@@ -57,7 +57,7 @@ class Dart extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->engine->{"display-name"},$this->engine->homepage,
$this->counter
);
}
......
......@@ -8,9 +8,9 @@ class Dmoznebel extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
public function __construct($name, \StdClass $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
parent::__construct($name, $engine, $metager);
}
public function loadResults($result)
......@@ -36,7 +36,7 @@ class Dmoznebel extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->engine->{"display-name"},$this->engine->homepage,
$this->counter
);
}
......
......@@ -8,9 +8,9 @@ class Dmoznebel_int extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
public function __construct($name, \StdClass $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
parent::__construct($name, $engine, $metager);
}
public function loadResults($result)
......@@ -28,7 +28,7 @@ class Dmoznebel_int extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->engine->{"display-name"},$this->engine->homepage,
$this->counter
);
}
......
......@@ -9,9 +9,9 @@ class Ebay extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
public function __construct($name, \stdClass $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
parent::__construct($name, $engine, $metager);
}
public function loadResults($result)
......@@ -24,15 +24,15 @@ class Ebay extends Searchengine
}
$results = $content->{"searchResult"};
$count = 0;
$count = 0;
foreach ($results->{"item"} as $result) {
$title = $result->{"title"}->__toString();
$link = $result->{"viewItemURL"}->__toString();
$title = $result->{"title"}->__toString();
$link = $result->{"viewItemURL"}->__toString();
$anzeigeLink = $link;
$time = $result->{"listingInfo"}->{"endTime"}->__toString();
$time = date(DATE_RFC2822, strtotime($time));
$price = intval($result->{"sellingStatus"}->{"convertedCurrentPrice"}->__toString()) * 100;
$descr = "Preis: " . $result->{"sellingStatus"}->{"convertedCurrentPrice"}->__toString() . " €";
$time = $result->{"listingInfo"}->{"endTime"}->__toString();
$time = date(DATE_RFC2822, strtotime($time));
$price = intval($result->{"sellingStatus"}->{"convertedCurrentPrice"}->__toString()) * 100;
$descr = "Preis: " . $result->{"sellingStatus"}->{"convertedCurrentPrice"}->__toString() . " €";
$descr .= ", Versandkosten: " . $result->{"shippingInfo"}->{"shippingServiceCost"}->__toString() . " €";
if (isset($result->{"listingInfo"}->{"listingType"})) {
$descr .= ", Auktionsart: " . $result->{"listingInfo"}->{"listingType"}->__toString();
......@@ -51,11 +51,11 @@ class Ebay extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->engine->{"display-name"}, $this->engine->homepage,
$this->counter,
['partnershop' => false,
'price' => $price,
'image' => $image]
'price' => $price,
'image' => $image]
);
$count++;
}
......
......@@ -9,9 +9,9 @@ class Ecoshopper extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
public function __construct($name, \StdClass $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
parent::__construct($name, $engine, $metager);
}
public function loadResults($result)
......@@ -49,7 +49,7 @@ class Ecoshopper extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->engine->{"display-name"},$this->engine->homepage,
$this->counter,
['partnershop' => false,
'image' => $image]
......
......@@ -9,9 +9,9 @@ class Europeana extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
public function __construct($name, \StdClass $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
parent::__construct($name, $engine, $metager);
}
public function loadResults($result)
......@@ -42,7 +42,7 @@ class Europeana extends Searchengine
$link,
$anzeigeLink,
$descr,
$this->displayName,$this->homepage,
$this->engine->{"display-name"},$this->engine->homepage,
$this->counter,
['image' => $image]
);
......
......@@ -9,9 +9,9 @@ class Exalead extends Searchengine
{
public $results = [];
public function __construct(\SimpleXMLElement $engine, \App\MetaGer $metager)
public function __construct($name, \StdClass $engine, \App\MetaGer $metager)
{
parent::__construct($engine, $metager);
parent::__construct($name, $engine, $metager);
}
public function loadResults($result)
......@@ -22,8 +22,8 @@ class Exalead extends Searchengine
if (!$content) {
return;