// yt - A fully featured command line YouTube client // // Copyright (C) 2024 Benedikt Peetz // SPDX-License-Identifier: GPL-3.0-or-later // // This file is part of Yt. // // You should have received a copy of the License along with this program. // If not, see . use std::{collections::HashMap, process::Stdio, str::FromStr}; use anyhow::{Context, Ok, Result}; use chrono::{DateTime, Utc}; use log::{error, info, warn}; use tokio::{ io::{AsyncBufReadExt, BufReader}, process::Command, }; use url::Url; use yt_dlp::{unsmuggle_url, wrapper::info_json::InfoJson}; use crate::{ app::App, storage::{ subscriptions::{get_subscriptions, Subscription}, video_database::{ extractor_hash::ExtractorHash, getters::get_all_hashes, setters::add_video, Video, VideoStatus, }, }, }; pub async fn update( app: &App, max_backlog: u32, subs_to_update: Vec, _concurrent_processes: usize, ) -> Result<()> { let subscriptions = get_subscriptions(&app).await?; let mut back_subs: HashMap = HashMap::new(); let mut urls: Vec = vec![]; for (name, sub) in subscriptions.0 { if subs_to_update.contains(&name) || subs_to_update.is_empty() { urls.push(sub.url.to_string()); back_subs.insert(sub.url.clone(), sub); } else { info!( "Not updating subscription '{}' as it was not specified", name ); } } let mut child = Command::new("./python_update/raw_update.py") .arg(max_backlog.to_string()) .args(&urls) .stdout(Stdio::piped()) .stderr(Stdio::null()) .stdin(Stdio::null()) .spawn() .context("Failed to call python3 update_raw")?; let mut out = BufReader::new( child .stdout .take() .expect("Should be able to take child stdout"), ) .lines(); let hashes = get_all_hashes(app).await?; while let Some(line) = out.next_line().await? { // use tokio::{fs::File, io::AsyncWriteExt}; // let mut output = File::create("output.json").await?; // output.write(line.as_bytes()).await?; // output.flush().await?; // output.sync_all().await?; // drop(output); let output_json: HashMap = serde_json::from_str(&line).expect("This should be valid json"); for (url, value) in output_json { let sub = back_subs.get(&url).expect("This was stored before"); process_subscription(app, sub, value, &hashes).await? } } let out = child.wait().await?; if out.success() { error!("A yt update-once invokation failed for all subscriptions.") } Ok(()) } async fn process_subscription( app: &App, sub: &Subscription, entry: InfoJson, hashes: &Vec, ) -> Result<()> { macro_rules! unwrap_option { ($option:expr) => { match $option { Some(x) => x, None => anyhow::bail!(concat!( "Expected a value, but '", stringify!($option), "' is None!" )), } }; } let publish_date = if let Some(date) = &entry.upload_date { let year: u32 = date .chars() .take(4) .collect::() .parse() .expect("Should work."); let month: u32 = date .chars() .skip(4) .take(2) .collect::() .parse() .expect("Should work"); let day: u32 = date .chars() .skip(6) .take(2) .collect::() .parse() .expect("Should work"); let date_string = format!("{year:04}-{month:02}-{day:02}T00:00:00Z"); Some( DateTime::::from_str(&date_string) .expect("This should always work") .timestamp(), ) } else { warn!( "The video '{}' lacks it's upload date!", unwrap_option!(&entry.title) ); None }; let thumbnail_url = match (&entry.thumbnails, &entry.thumbnail) { (None, None) => None, (None, Some(thumbnail)) => Some(thumbnail.to_owned()), // TODO: The algorithm is not exactly the best <2024-05-28> (Some(thumbnails), None) => Some( thumbnails .get(0) .expect("At least one should exist") .url .clone(), ), (Some(_), Some(thumnail)) => Some(thumnail.to_owned()), }; let url = { let smug_url: url::Url = unwrap_option!(entry.webpage_url.clone()); unsmuggle_url(smug_url)? }; let extractor_hash = blake3::hash(url.as_str().as_bytes()); if hashes.contains(&extractor_hash) { // We already stored the video information println!( "(Ignoring duplicated video from: '{}' -> '{}')", sub.name, unwrap_option!(entry.title) ); return Ok(()); } else { let video = Video { cache_path: None, description: entry.description.clone(), duration: entry.duration, extractor_hash: ExtractorHash::from_hash(extractor_hash), last_status_change: Utc::now().timestamp(), parent_subscription_name: Some(sub.name.clone()), priority: 0, publish_date, status: VideoStatus::Pick, status_change: false, thumbnail_url, title: unwrap_option!(entry.title.clone()), url, }; println!("{}", video.to_color_display()); add_video(app, video).await?; } Ok(()) }