Yahoo client only fetches the latest news within a specific interval.

Author: metalwhale

chloria-backend/chloria-job/src/infrastructure/news_fetcher/yahoo.rs

@@ -36,10 +36,11 @@ struct NewsResponseChannelItem {
 
 pub(crate) struct YahooClient {
     providers: Vec<String>,
+    interval: i64,
 }
 
 impl YahooClient {
-    pub(crate) fn new() -> Self {
+    pub(crate) fn new(interval: i64) -> Self {
         Self {
             // TODO: Find a better way to retrieve the list of providers
             providers: vec![
@@ -742,6 +743,7 @@ impl YahooClient {
             .iter()
             .map(|p| p.to_string())
             .collect(),
+            interval,
         }
     }
 
@@ -767,7 +769,7 @@ impl YahooClient {
                 Some(pub_date) => match DateTime::parse_from_rfc2822(&pub_date) {
                     Ok(published_time) => {
                         let published_time: DateTime<Local> = published_time.into();
-                        if (now - published_time).num_days() > 0 {
+                        if (now - published_time).num_hours() >= self.interval {
                             continue;
                         }
                         Some(published_time)

chloria-backend/chloria-job/src/main.rs

@@ -42,6 +42,7 @@ async fn main() -> Result<()> {
         .collect::<Vec<_>>();
     let chloria_origin_bucket_name = env::var("CHLORIA_ORIGIN_BUCKET_NAME")?;
     let chloria_case_permits_num = env::var("CHLORIA_CASE_PERMITS_NUM")?.parse::<usize>().unwrap_or(10);
+    let chloria_job_interval = env::var("CHLORIA_JOB_INTERVAL")?.parse()?; // In hours
     env_logger::init_from_env(Env::new().filter("CHLORIA_LOG"));
     // Initialize infrastructure
     let mut news_fetchers: Vec<Arc<dyn NewsFetcher>> = vec![];
@@ -52,7 +53,7 @@ async fn main() -> Result<()> {
         }
     }
     if chloria_news_fetchers.contains(&"yahoo".to_string()) {
-        let yahoo_client = YahooClient::new();
+        let yahoo_client = YahooClient::new(chloria_job_interval);
         news_fetchers.push(Arc::new(yahoo_client));
     }
     let reqwest_tool = ReqwestTool::new();

deployment-local/compose.yaml

@@ -57,6 +57,7 @@ services:
       - CHLORIA_NEWS_FETCHERS=yahoo
       - CHLORIA_ORIGIN_BUCKET_NAME=origin
       - CHLORIA_CASE_PERMITS_NUM=10
+      - CHLORIA_JOB_INTERVAL=12
       - CHLORIA_LOG=info
     volumes:
       - ../:/usr/src/chloria/