Fix: Allow negative values for remaining results. Show errors if fetching fails.

Author: sontdhust

chloria-backend/chloria-job/src/infrastructure/newsdata.rs

@@ -1,8 +1,8 @@
-use anyhow::Result;
+use anyhow::{Context, Result};
 use async_trait::async_trait;
 use chrono::NaiveDateTime;
 use chrono_tz::Tz;
-use log::info;
+use log::{error, info};
 use reqwest::Client;
 use serde::Deserialize;
 
@@ -51,7 +51,8 @@ impl NewsdataClient {
             url.push_str(&format!("&page={page}"));
         }
         let response_text = Client::new().get(url).send().await?.text().await?;
-        let news_response: NewsResponse = serde_json::from_str(&response_text)?;
+        let news_response: NewsResponse = serde_json::from_str(&response_text)
+            .with_context(|| format!("Deserializing the response text: {}", response_text))?;
         let mut articles = vec![];
         for result in news_response.results.into_iter() {
             let published_time = if let (Some(pub_date), Some(pub_date_tz)) = (result.pub_date, result.pub_date_tz) {
@@ -102,30 +103,34 @@ impl NewsFetcher for NewsdataClient {
             } {
                 break;
             }
-            if let Ok((total_results, page_articles, next_page)) = self.fetch_page(page).await {
-                info!(
-                    "total_results={}, page_index={}, page_articles.len={}, next_page={:?}",
-                    total_results,
-                    page_index,
-                    page_articles.len(),
-                    next_page,
-                );
-                remaining_results_num = Some(
-                    match remaining_results_num {
-                        Some(remaining_results_num) => remaining_results_num,
-                        None => total_results,
-                    } - page_articles.len() as u16,
-                );
-                for article in page_articles {
-                    output.push(handler(article));
+            match self.fetch_page(page).await {
+                Ok((total_results, page_articles, next_page)) => {
+                    info!(
+                        "page_index={}, total_results={}, page_articles.len={}, next_page={:?}",
+                        page_index,
+                        total_results,
+                        page_articles.len(),
+                        next_page,
+                    );
+                    remaining_results_num = Some(
+                        match remaining_results_num {
+                            Some(remaining_results_num) => remaining_results_num,
+                            None => total_results as i32,
+                        } - page_articles.len() as i32,
+                    );
+                    for article in page_articles {
+                        output.push(handler(article));
+                    }
+                    match next_page {
+                        Some(next_page) => page = Some(next_page),
+                        None => break,
+                    };
+                    page_index += 1;
+                }
+                Err(error) => {
+                    error!("page_index={}, error={}", page_index, error);
+                    break;
                 }
-                match next_page {
-                    Some(next_page) => page = Some(next_page),
-                    None => break,
-                };
-                page_index += 1;
-            } else {
-                break;
             }
         }
         output