added pagination

scraper/marginalia.php

@@ -220,6 +220,7 @@ class marginalia{
 			"related" => []
 		];
 		
+		// API scraper
 		if(config::MARGINALIA_API_KEY !== null){
 			
 			try{
@@ -263,7 +264,29 @@ class marginalia{
 			return $out;
 		}
 		
-		// no more cloudflare!! Parse html by default
+		// HTML parser
+		$proxy = $this->backend->get_ip();
+		
+		if($get["npt"]){
+			
+			[$params, $proxy] =
+				$this->backend->get(
+					$get["npt"],
+					"web"
+				);
+			
+			try{
+				$html =
+					$this->get(
+						$proxy,
+						"https://search.marginalia.nu/search?" . $params
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to get HTML");
+			}
+			
+		}else{
 			$params = [
 				"query" => $search
 			];
@@ -284,7 +307,7 @@ class marginalia{
 			try{
 				$html =
 					$this->get(
-					$this->backend->get_ip(),
+						$proxy,
 						"https://search.marginalia.nu/search",
 						$params
 					);
@@ -292,6 +315,7 @@ class marginalia{
 				
 				throw new Exception("Failed to get HTML");
 			}
+		}
 		
 		$this->fuckhtml->load($html);
 		
@@ -387,6 +411,65 @@ class marginalia{
 			];
 		}
 		
+		// get next page
+		$this->fuckhtml->load($html);
+		
+		$pagination =
+			$this->fuckhtml
+			->getElementsByAttributeValue(
+				"aria-label",
+				"pagination",
+				"nav"
+			);
+		
+		if(count($pagination) === 0){
+			
+			// no pagination
+			return $out;
+		}
+		
+		$this->fuckhtml->load($pagination[0]);
+		
+		$pages =
+			$this->fuckhtml
+			->getElementsByClassName(
+				"page-link",
+				"a"
+			);
+		
+		$found_current_page = false;
+		
+		foreach($pages as $page){
+			
+			if(
+				stripos(
+					$page["attributes"]["class"],
+					"active"
+				) !== false
+			){
+				
+				$found_current_page = true;
+				continue;
+			}
+			
+			if($found_current_page){
+				
+				// we found current page index, and we iterated over
+				// the next page <a>
+				
+				$out["npt"] =
+					$this->backend->store(
+						parse_url(
+							$page["attributes"]["href"],
+							PHP_URL_QUERY
+						),
+						"web",
+						$proxy
+					);
+				break;
+			}
+		}
+		
 		return $out;
 	}
 }