chore: Initial Commit

This repository was migrated out of my nixos-config.

1 files changed, 207 insertions, 0 deletions

diff --git a/src/update/mod.rs b/src/update/mod.rs
new file mode 100644
index 0000000..9128bf7
--- /dev/null
+++ b/src/update/mod.rs
@@ -0,0 +1,207 @@
+// yt - A fully featured command line YouTube client
+//
+// Copyright (C) 2024 Benedikt Peetz <benedikt.peetz@b-peetz.de>
+// SPDX-License-Identifier: GPL-3.0-or-later
+//
+// This file is part of Yt.
+//
+// You should have received a copy of the License along with this program.
+// If not, see <https://www.gnu.org/licenses/gpl-3.0.txt>.
+
+use std::{collections::HashMap, process::Stdio, str::FromStr};
+
+use anyhow::{Context, Ok, Result};
+use chrono::{DateTime, Utc};
+use log::{error, info, warn};
+use tokio::{
+    io::{AsyncBufReadExt, BufReader},
+    process::Command,
+};
+use url::Url;
+use yt_dlp::{unsmuggle_url, wrapper::info_json::InfoJson};
+
+use crate::{
+    app::App,
+    storage::{
+        subscriptions::{get_subscriptions, Subscription},
+        video_database::{
+            extractor_hash::ExtractorHash, getters::get_all_hashes, setters::add_video, Video,
+            VideoStatus,
+        },
+    },
+};
+
+pub async fn update(
+    app: &App,
+    max_backlog: u32,
+    subs_to_update: Vec<String>,
+    _concurrent_processes: usize,
+) -> Result<()> {
+    let subscriptions = get_subscriptions(&app).await?;
+    let mut back_subs: HashMap<Url, Subscription> = HashMap::new();
+
+    let mut urls: Vec<String> = vec![];
+    for (name, sub) in subscriptions.0 {
+        if subs_to_update.contains(&name) || subs_to_update.is_empty() {
+            urls.push(sub.url.to_string());
+            back_subs.insert(sub.url.clone(), sub);
+        } else {
+            info!(
+                "Not updating subscription '{}' as it was not specified",
+                name
+            );
+        }
+    }
+
+    let mut child = Command::new("./python_update/raw_update.py")
+        .arg(max_backlog.to_string())
+        .args(&urls)
+        .stdout(Stdio::piped())
+        .stderr(Stdio::null())
+        .stdin(Stdio::null())
+        .spawn()
+        .context("Failed to call python3 update_raw")?;
+
+    let mut out = BufReader::new(
+        child
+            .stdout
+            .take()
+            .expect("Should be able to take child stdout"),
+    )
+    .lines();
+
+    let hashes = get_all_hashes(app).await?;
+
+    while let Some(line) = out.next_line().await? {
+        // use tokio::{fs::File, io::AsyncWriteExt};
+        // let mut output = File::create("output.json").await?;
+        // output.write(line.as_bytes()).await?;
+        // output.flush().await?;
+        // output.sync_all().await?;
+        // drop(output);
+
+        let output_json: HashMap<Url, InfoJson> =
+            serde_json::from_str(&line).expect("This should be valid json");
+
+        for (url, value) in output_json {
+            let sub = back_subs.get(&url).expect("This was stored before");
+            process_subscription(app, sub, value, &hashes).await?
+        }
+    }
+
+    let out = child.wait().await?;
+    if out.success() {
+        error!("A yt update-once invokation failed for all subscriptions.")
+    }
+
+    Ok(())
+}
+
+async fn process_subscription(
+    app: &App,
+    sub: &Subscription,
+    entry: InfoJson,
+    hashes: &Vec<blake3::Hash>,
+) -> Result<()> {
+    macro_rules! unwrap_option {
+        ($option:expr) => {
+            match $option {
+                Some(x) => x,
+                None => anyhow::bail!(concat!(
+                    "Expected a value, but '",
+                    stringify!($option),
+                    "' is None!"
+                )),
+            }
+        };
+    }
+
+    let publish_date = if let Some(date) = &entry.upload_date {
+        let year: u32 = date
+            .chars()
+            .take(4)
+            .collect::<String>()
+            .parse()
+            .expect("Should work.");
+        let month: u32 = date
+            .chars()
+            .skip(4)
+            .take(2)
+            .collect::<String>()
+            .parse()
+            .expect("Should work");
+        let day: u32 = date
+            .chars()
+            .skip(6)
+            .take(2)
+            .collect::<String>()
+            .parse()
+            .expect("Should work");
+
+        let date_string = format!("{year:04}-{month:02}-{day:02}T00:00:00Z");
+        Some(
+            DateTime::<Utc>::from_str(&date_string)
+                .expect("This should always work")
+                .timestamp(),
+        )
+    } else {
+        warn!(
+            "The video '{}' lacks it's upload date!",
+            unwrap_option!(&entry.title)
+        );
+        None
+    };
+
+    let thumbnail_url = match (&entry.thumbnails, &entry.thumbnail) {
+        (None, None) => None,
+        (None, Some(thumbnail)) => Some(thumbnail.to_owned()),
+
+        // TODO: The algorithm is not exactly the best <2024-05-28>
+        (Some(thumbnails), None) => Some(
+            thumbnails
+                .get(0)
+                .expect("At least one should exist")
+                .url
+                .clone(),
+        ),
+        (Some(_), Some(thumnail)) => Some(thumnail.to_owned()),
+    };
+
+    let url = {
+        let smug_url: url::Url = unwrap_option!(entry.webpage_url.clone());
+        unsmuggle_url(smug_url)?
+    };
+
+    let extractor_hash = blake3::hash(url.as_str().as_bytes());
+
+    if hashes.contains(&extractor_hash) {
+        // We already stored the video information
+        println!(
+            "(Ignoring duplicated video from: '{}' -> '{}')",
+            sub.name,
+            unwrap_option!(entry.title)
+        );
+        return Ok(());
+    } else {
+        let video = Video {
+            cache_path: None,
+            description: entry.description.clone(),
+            duration: entry.duration,
+            extractor_hash: ExtractorHash::from_hash(extractor_hash),
+            last_status_change: Utc::now().timestamp(),
+            parent_subscription_name: Some(sub.name.clone()),
+            priority: 0,
+            publish_date,
+            status: VideoStatus::Pick,
+            status_change: false,
+            thumbnail_url,
+            title: unwrap_option!(entry.title.clone()),
+            url,
+        };
+
+        println!("{}", video.to_color_display());
+        add_video(app, video).await?;
+    }
+
+    Ok(())
+}