fix wikipedia crash

scraper/ddg.php

@@ -1046,20 +1046,38 @@ class ddg{
 				
 				if(isset($json["Abstract"])){
 					
-					$description[] =
+					$description = $this->parse_rich_text($json["Abstract"]);
-						[
+				}
-							"type" => "text",
+				
-							"value" => $json["Abstract"]
+				if(
-						];
+					!isset($json["Image"]) ||
+					$json["Image"] == "" ||
+					$json["Image"] === null ||
+					$json["Image"] == "https://duckduckgo.com/i/"
+				){
+					
+					$image = null;
+				}else{
+					
+					if(
+						preg_match(
+							'/^https?:\/\//',
+							$json["Image"]
+						)
+					){
+						
+						$image = $json["Image"];
+					}else{
+						
+						$image = "https://duckduckgo.com" . $json["Image"];
+					}
 				}
 				
 				$out["answer"][] = [
 					"title" => $json["Heading"],
 					"description" => $description,
 					"url" => $json["AbstractURL"],
-					"thumb" =>
+					"thumb" => $image,
-						(!isset($json["Image"]) || $json["Image"] == "" || $json["Image"] === null) ?
-						null : "https://duckduckgo.com" . $json["Image"],
 					"table" => $table,
 					"sublink" => $sublinks
 				];
@@ -1072,11 +1090,11 @@ class ddg{
 		}
 		
 		//
-		// Get wordnik definition
+		// Parse additional data endpoints
 		//
 		//nrj('/js/spice/dictionary/definition/create', null, null, null, null, 'dictionary_definition');
 		
-		preg_match(
+		preg_match_all(
 			'/nrj\(\s*\'([^\']+)\'/',
 			$js,
 			$nrj
@@ -1084,11 +1102,14 @@ class ddg{
 		
 		if(isset($nrj[1])){
 			
-			$nrj = $nrj[1];
+			foreach($nrj[1] as $potential_endpoint){
 				
+				//
+				// Probe for wordnik definition
+				//
 				preg_match(
 					'/\/js\/spice\/dictionary\/definition\/([^\/]+)/',
-				$nrj,
+					$potential_endpoint,
 					$word
 				);
 				
@@ -1314,6 +1335,87 @@ class ddg{
 						];
 					}
 				}
+				
+				//
+				// Parse stackoverflow answer
+				//
+				if(
+					preg_match(
+						'/^\/a\.js.*src_id=stack_overflow/',
+						$potential_endpoint
+					)
+				){
+					
+					// found stackoverflow answer
+					try{
+						$json =
+							$this->get(
+								$proxy,
+								"https://duckduckgo.com" . $potential_endpoint,
+								[],
+								ddg::req_xhr
+							);
+						
+					}catch(Exception $e){
+						
+						// fail gracefully
+						return $out;
+					}
+					
+					$json = explode("DDG.duckbar.add_array(", $json, 2);
+					
+					if(count($json) === 2){
+						
+						$json =
+							json_decode(
+								$this->fuckhtml
+								->extract_json(
+									$json[1]
+								),
+								true
+							);
+						
+						if(
+							$json !== null &&
+							isset($json[0]["data"])
+						){
+							
+							$json = $json[0]["data"];
+							
+							foreach($json as $answer){
+								
+								if(isset($answer["Heading"])){
+									
+									$title = $answer["Heading"];
+								}elseif(isset($answer["title"])){
+									
+									$title = $answer["title"];
+								}else{
+									
+									$title = null;
+								}
+								
+								if(
+									$title !== null &&
+									isset($answer["Abstract"])
+								){
+									
+									$description = $this->parse_rich_text($answer["Abstract"]);
+									
+									$out["answer"][] = [
+										"title" => $title,
+										"description" => $description,
+										"url" => $answer["AbstractURL"],
+										"thumb" => null,
+										"table" => [],
+										"sublink" => []
+									];
+								}
+							}
+						}
+					}
+				}
+			}
 		}
 		
 		return $out;
@@ -1841,6 +1943,146 @@ class ddg{
 		return $out;
 	}
 	
+	private function parse_rich_text($html){
+		
+		$description = [];
+		
+		// pre-process the html, remove useless elements
+		$html =
+			strip_tags(
+				$html,
+				[
+					"h1", "h2", "h3", "h4", "h5", "h6", "h7",
+					"pre", "code"
+				]
+			);
+		
+		$html =
+			preg_replace(
+				'/<(\/?)pre *[^>]*>\s*<\/?code *[^>]*>/i',
+				'<$1pre>',
+				$html
+			);
+		
+		$this->fuckhtml->load($html);
+		
+		$tags =
+			$this->fuckhtml
+			->getElementsByTagName(
+				"*"
+			);
+		
+		if(count($tags) === 0){
+			
+			$description[] = [
+				"type" => "text",
+				"value" =>
+					trim(
+						$this->fuckhtml
+						->getTextContent(
+							$html,
+							true,
+							false
+						)
+					)
+			];
+		}else{
+			
+			$start = 0;
+			$was_code_block = true;
+			foreach($tags as $tag){
+				
+				$text =
+					$this->fuckhtml
+					->getTextContent(
+						substr(
+							$html,
+							$start,
+							$tag["startPos"] - $start
+						),
+						true,
+						false
+					);
+				
+				if($was_code_block){
+					
+					$text = ltrim($text);
+					$was_code_block = false;
+				}
+				
+				$description[] = [
+					"type" => "text",
+					"value" => $text
+				];
+				
+				switch($tag["tagName"]){
+					
+					case "pre":
+						$append = "code";
+						$was_code_block = true;
+						$c = count($description) - 1;
+						$description[$c]["value"] =
+							rtrim($description[$c]["value"]);
+						break;
+					
+					case "code":
+						$append = "inline_code";
+						$c = count($description) - 1;
+						$description[$c]["value"] =
+							rtrim($description[$c]["value"]) . " ";
+						break;
+					
+					case "h1":
+					case "h2":
+					case "h3":
+					case "h4":
+					case "h5":
+					case "h6":
+					case "h7":
+						$append = "title";
+						$c = count($description) - 1;
+						$description[$c]["value"] =
+							rtrim($description[$c]["value"]);
+						break;
+				}
+				
+				$description[] = [
+					"type" => $append,
+					"value" =>
+						trim(
+							$this->fuckhtml
+							->getTextContent(
+								$tag,
+								true,
+								false
+							)
+						)
+				];
+				
+				$start = $tag["endPos"];
+			}
+			
+			// shit out remainder
+			$description[] = [
+				"type" => "text",
+				"value" =>
+					trim(
+						$this->fuckhtml
+						->getTextContent(
+							substr(
+								$html,
+								$start
+							),
+							true,
+							false
+						)
+					)
+			];
+		}
+		
+		return $description;
+	}
+	
 	private function titledots($title){
 		
 		$substr = substr($title, -3);