From 1debeb77f7986de1b659dcfdc442de6415e1d9f5 Mon Sep 17 00:00:00 2001 From: Benedikt Peetz Date: Wed, 21 Aug 2024 10:49:23 +0200 Subject: chore: Initial Commit This repository was migrated out of my nixos-config. --- src/storage/video_database/downloader.rs | 210 +++++++++++++++++ src/storage/video_database/extractor_hash.rs | 151 ++++++++++++ src/storage/video_database/getters.rs | 339 +++++++++++++++++++++++++++ src/storage/video_database/mod.rs | 170 ++++++++++++++ src/storage/video_database/schema.sql | 56 +++++ src/storage/video_database/setters.rs | 270 +++++++++++++++++++++ 6 files changed, 1196 insertions(+) create mode 100644 src/storage/video_database/downloader.rs create mode 100644 src/storage/video_database/extractor_hash.rs create mode 100644 src/storage/video_database/getters.rs create mode 100644 src/storage/video_database/mod.rs create mode 100644 src/storage/video_database/schema.sql create mode 100644 src/storage/video_database/setters.rs (limited to 'src/storage/video_database') diff --git a/src/storage/video_database/downloader.rs b/src/storage/video_database/downloader.rs new file mode 100644 index 0000000..c04ab8d --- /dev/null +++ b/src/storage/video_database/downloader.rs @@ -0,0 +1,210 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use std::path::{Path, PathBuf}; + +use anyhow::Result; +use log::debug; +use sqlx::query; +use url::Url; + +use crate::{app::App, storage::video_database::VideoStatus}; + +use super::{ExtractorHash, Video}; + +/// Returns to next video which should be downloaded. This respects the priority assigned by select. +/// It does not return videos, which are already cached. +pub async fn get_next_uncached_video(app: &App) -> Result> { + let status = VideoStatus::Watch.as_db_integer(); + + let result = query!( + r#" + SELECT * + FROM videos + WHERE status = ? AND cache_path IS NULL + ORDER BY priority ASC + LIMIT 1; + "#, + status + ) + .fetch_one(&app.database) + .await; + + if let Err(sqlx::Error::RowNotFound) = result { + Ok(None) + } else { + let base = result?; + + let thumbnail_url = if let Some(url) = &base.thumbnail_url { + Some(Url::parse(&url)?) + } else { + None + }; + + let status_change = if base.status_change == 1 { + true + } else { + assert_eq!(base.status_change, 0, "Can only be 1 or 0"); + false + }; + + let video = Video { + cache_path: base.cache_path.as_ref().map(|val| PathBuf::from(val)), + description: base.description.clone(), + duration: base.duration, + extractor_hash: ExtractorHash::from_hash( + base.extractor_hash + .parse() + .expect("The hash in the db should be valid"), + ), + last_status_change: base.last_status_change, + parent_subscription_name: base.parent_subscription_name.clone(), + priority: base.priority, + publish_date: base.publish_date, + status: VideoStatus::from_db_integer(base.status), + status_change, + thumbnail_url, + title: base.title.clone(), + url: Url::parse(&base.url)?, + }; + + Ok(Some(video)) + } +} + +/// Returns to next video which can be watched (i.e. is cached). +/// This respects the priority assigned by select. +pub async fn get_next_video_watchable(app: &App) -> Result> { + let result = query!( + r#" + SELECT * + FROM videos + WHERE status = 'Watching' AND cache_path IS NOT NULL + ORDER BY priority ASC + LIMIT 1; + "# + ) + .fetch_one(&app.database) + .await; + + if let Err(sqlx::Error::RowNotFound) = result { + Ok(None) + } else { + let base = result?; + + let thumbnail_url = if let Some(url) = &base.thumbnail_url { + Some(Url::parse(&url)?) + } else { + None + }; + + let status_change = if base.status_change == 1 { + true + } else { + assert_eq!(base.status_change, 0, "Can only be 1 or 0"); + false + }; + + let video = Video { + cache_path: base.cache_path.as_ref().map(|val| PathBuf::from(val)), + description: base.description.clone(), + duration: base.duration, + extractor_hash: ExtractorHash::from_hash( + base.extractor_hash + .parse() + .expect("The db extractor_hash should be valid blake3 hash"), + ), + last_status_change: base.last_status_change, + parent_subscription_name: base.parent_subscription_name.clone(), + priority: base.priority, + publish_date: base.publish_date, + status: VideoStatus::from_db_integer(base.status), + status_change, + thumbnail_url, + title: base.title.clone(), + url: Url::parse(&base.url)?, + }; + + Ok(Some(video)) + } +} + +/// Update the cached path of a video. Will be set to NULL if the path is None +/// This will also set the status to `Cached` when path is Some, otherwise it set's the status to +/// `Watch`. +pub async fn set_video_cache_path( + app: &App, + video: &ExtractorHash, + path: Option<&Path>, +) -> Result<()> { + if let Some(path) = path { + debug!( + "Setting cache path from '{}' to '{}'", + video.into_short_hash(app).await?, + path.display() + ); + + let path_str = path.display().to_string(); + let extractor_hash = video.hash().to_string(); + let status = VideoStatus::Cached.as_db_integer(); + + query!( + r#" + UPDATE videos + SET cache_path = ?, status = ? + WHERE extractor_hash = ?; + "#, + path_str, + status, + extractor_hash + ) + .execute(&app.database) + .await?; + + Ok(()) + } else { + debug!( + "Setting cache path from '{}' to NULL", + video.into_short_hash(app).await?, + ); + + let extractor_hash = video.hash().to_string(); + let status = VideoStatus::Watch.as_db_integer(); + + query!( + r#" + UPDATE videos + SET cache_path = NULL, status = ? + WHERE extractor_hash = ?; + "#, + status, + extractor_hash + ) + .execute(&app.database) + .await?; + + Ok(()) + } +} + +/// Returns the number of cached videos +pub async fn get_allocated_cache(app: &App) -> Result { + let count = query!( + r#" + SELECT COUNT(cache_path) as count + FROM videos + WHERE cache_path IS NOT NULL; +"#, + ) + .fetch_one(&app.database) + .await?; + + Ok(count.count as u32) +} diff --git a/src/storage/video_database/extractor_hash.rs b/src/storage/video_database/extractor_hash.rs new file mode 100644 index 0000000..3af4f60 --- /dev/null +++ b/src/storage/video_database/extractor_hash.rs @@ -0,0 +1,151 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +use std::{collections::HashMap, fmt::Display, str::FromStr}; + +use anyhow::{bail, Result}; +use blake3::Hash; +use log::debug; +use tokio::sync::OnceCell; + +use crate::{app::App, storage::video_database::getters::get_all_hashes}; + +static EXTRACTOR_HASH_LENGTH: OnceCell = OnceCell::const_new(); + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExtractorHash { + hash: Hash, +} + +#[derive(Debug, Clone)] +pub struct ShortHash(String); + +impl Display for ShortHash { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + self.0.fmt(f) + } +} + +#[derive(Debug, Clone)] +pub struct LazyExtractorHash { + value: ShortHash, +} + +impl FromStr for LazyExtractorHash { + type Err = anyhow::Error; + + fn from_str(s: &str) -> std::result::Result { + // perform some cheap validation + if s.len() > 64 { + bail!("A hash can only contain 64 bytes!"); + } + + Ok(Self { + value: ShortHash(s.to_owned()), + }) + } +} + +impl LazyExtractorHash { + /// Turn the [`LazyExtractorHash`] into the [`ExtractorHash`] + pub async fn realize(self, app: &App) -> Result { + ExtractorHash::from_short_hash(app, &self.value).await + } +} + +impl ExtractorHash { + pub fn from_hash(hash: Hash) -> Self { + Self { hash } + } + pub async fn from_short_hash(app: &App, s: &ShortHash) -> Result { + Ok(Self { + hash: Self::short_hash_to_full_hash(app, s).await?, + }) + } + + pub fn hash(&self) -> &Hash { + &self.hash + } + + pub async fn into_short_hash(&self, app: &App) -> Result { + let needed_chars = if let Some(needed_chars) = EXTRACTOR_HASH_LENGTH.get() { + debug!("Using cached char length: {}", needed_chars); + *needed_chars + } else { + let needed_chars = self.get_needed_char_len(app).await?; + debug!("Setting the needed has char lenght."); + EXTRACTOR_HASH_LENGTH + .set(needed_chars) + .expect("This should work at this stage"); + + needed_chars + }; + + debug!("Formatting a hash with char length: {}", needed_chars); + + Ok(ShortHash( + self.hash() + .to_hex() + .chars() + .into_iter() + .take(needed_chars) + .collect::(), + )) + } + + async fn short_hash_to_full_hash(app: &App, s: &ShortHash) -> Result { + let all_hashes = get_all_hashes(app).await?; + + let needed_chars = s.0.len(); + + for hash in all_hashes { + if &hash.to_hex()[..needed_chars] == s.0 { + return Ok(hash); + } + } + + bail!("Your shortend hash, does not match a real hash (this is probably a bug)!"); + } + + async fn get_needed_char_len(&self, app: &App) -> Result { + debug!("Calculating the needed hash char length"); + let all_hashes = get_all_hashes(app).await?; + + let all_char_vec_hashes = all_hashes + .into_iter() + .map(|hash| hash.to_hex().chars().collect::>()) + .collect::>>(); + + // This value should be updated later, if not rust will panic in the assertion. + let mut needed_chars: usize = 1000; + 'outer: for i in 1..64 { + let i_chars: Vec = all_char_vec_hashes + .iter() + .map(|vec| vec.iter().take(i).collect::()) + .collect(); + + let mut uniqnes_hashmap: HashMap = HashMap::new(); + for ch in i_chars { + if let Some(()) = uniqnes_hashmap.insert(ch, ()) { + // The key was already in the hash map, thus we have a duplicated char and need + // at least one char more + continue 'outer; + } + } + + needed_chars = i; + break 'outer; + } + + assert!(needed_chars <= 64, "Hashes are only 64 bytes long"); + + Ok(needed_chars) + } +} diff --git a/src/storage/video_database/getters.rs b/src/storage/video_database/getters.rs new file mode 100644 index 0000000..ca4164d --- /dev/null +++ b/src/storage/video_database/getters.rs @@ -0,0 +1,339 @@ +// yt - A fully featured command line YouTube client +// +// Copyright (C) 2024 Benedikt Peetz +// SPDX-License-Identifier: GPL-3.0-or-later +// +// This file is part of Yt. +// +// You should have received a copy of the License along with this program. +// If not, see . + +//! These functions interact with the storage db in a read-only way. They are added on-demaned (as +//! you could theoretically just could do everything with the `get_videos` function), as +//! performance or convince requires. +use std::{fs::File, path::PathBuf}; + +use anyhow::{bail, Context, Result}; +use blake3::Hash; +use log::debug; +use sqlx::{query, QueryBuilder, Row, Sqlite}; +use url::Url; +use yt_dlp::wrapper::info_json::InfoJson; + +use crate::{ + app::App, + storage::{ + subscriptions::Subscription, + video_database::{extractor_hash::ExtractorHash, Video}, + }, +}; + +use super::{MpvOptions, VideoOptions, VideoStatus, YtDlpOptions}; + +macro_rules! video_from_record { + ($record:expr) => { + let thumbnail_url = if let Some(url) = &$record.thumbnail_url { + Some(Url::parse(&url)?) + } else { + None + }; + + Ok(Video { + cache_path: $record.cache_path.as_ref().map(|val| PathBuf::from(val)), + description: $record.description.clone(), + duration: $record.duration, + extractor_hash: ExtractorHash::from_hash( + $record + .extractor_hash + .parse() + .expect("The db hash should be a valid blake3 hash"), + ), + last_status_change: $record.last_status_change, + parent_subscription_name: $record.parent_subscription_name.clone(), + publish_date: $record.publish_date, + status: VideoStatus::from_db_integer($record.status), + thumbnail_url, + title: $record.title.clone(), + url: Url::parse(&$record.url)?, + priority: $record.priority, + status_change: if $record.status_change == 1 { + true + } else { + assert_eq!($record.status_change, 0); + false + }, + }) + }; +} + +/// Get the lines to display at the selection file +/// [`changing` = true]: Means that we include *only* videos, that have the `status_changing` flag set +/// [`changing` = None]: Means that we include *both* videos, that have the `status_changing` flag set and not set +pub async fn get_videos( + app: &App, + allowed_states: &[VideoStatus], + changing: Option, +) -> Result> { + let mut qb: QueryBuilder = QueryBuilder::new( + "\ + SELECT * + FROM videos + WHERE status IN ", + ); + + qb.push("("); + allowed_states + .iter() + .enumerate() + .for_each(|(index, state)| { + qb.push("'"); + qb.push(state.as_db_integer()); + qb.push("'"); + + if index != allowed_states.len() - 1 { + qb.push(","); + } + }); + qb.push(")"); + + if let Some(val) = changing { + if val { + qb.push(" AND status_change = 1"); + } else { + qb.push(" AND status_change = 0"); + } + } + + qb.push("\n ORDER BY priority DESC;"); + + debug!("Will run: \"{}\"", qb.sql()); + + let videos = qb.build().fetch_all(&app.database).await.with_context(|| { + format!( + "Failed to query videos with states: '{}'", + allowed_states.iter().fold(String::new(), |mut acc, state| { + acc.push(' '); + acc.push_str(&state.as_str()); + acc + }), + ) + })?; + + let real_videos: Vec