Lines of src/core.rs from check-in a2880e5100 that are changed by the sequence of edits moving toward check-in e624ef9d66:
a2880e5100 2025-04-19 1: use anyhow::{anyhow, bail, Context, Result}; a2880e5100 2025-04-19 2: use async_std::task; a2880e5100 2025-04-19 3: use chrono::DateTime; a2880e5100 2025-04-19 4: use sqlx::postgres::PgPoolOptions; a2880e5100 2025-04-19 5: use teloxide::{ a2880e5100 2025-04-19 6: Bot, a2880e5100 2025-04-19 7: payloads::SendMessage, a2880e5100 2025-04-19 8: requests::Requester, a2880e5100 2025-04-19 9: types::{ a2880e5100 2025-04-19 10: Me, a2880e5100 2025-04-19 11: UserId, a2880e5100 2025-04-19 12: }, a2880e5100 2025-04-19 13: }; a2880e5100 2025-04-19 14: use thiserror::Error; 15: 16: use std::{ 17: borrow::Cow, 18: collections::{ 19: BTreeMap, 20: HashSet, 21: }, 22: num::TryFromIntError, 23: sync::{ 24: Arc, 25: Mutex 26: }, 27: }; 28: 29: #[derive(Error, Debug)] 30: pub enum RssError { 31: // #[error(transparent)] 32: // Tg(#[from] TgError), 33: #[error(transparent)] 34: Int(#[from] TryFromIntError), 35: } 36: 37: #[derive(Clone)] 38: pub struct Core { a2880e5100 2025-04-19 39: owner_chat: UserId, 40: pub tg: Bot, a2880e5100 2025-04-19 41: pub my: Me, 42: pool: sqlx::Pool<sqlx::Postgres>, 43: sources: Arc<Mutex<HashSet<Arc<i32>>>>, 44: http_client: reqwest::Client, 45: } 46: 47: impl Core { a2880e5100 2025-04-19 48: pub fn new(settings: config::Config) -> Result<Arc<Core>> { a2880e5100 2025-04-19 49: let owner: u64 = settings.get_int("owner")?.try_into()?; 50: let api_key = settings.get_string("api_key")?; a2880e5100 2025-04-19 51: let tg = Bot::new(api_key); a2880e5100 2025-04-19 52: let tg_cloned = tg.clone(); 53: 54: let mut client = reqwest::Client::builder(); 55: if let Ok(proxy) = settings.get_string("proxy") { 56: let proxy = reqwest::Proxy::all(proxy)?; 57: client = client.proxy(proxy); 58: } 59: let http_client = client.build()?; 60: let core = Arc::new(Core { 61: tg, a2880e5100 2025-04-19 62: my: task::block_on(async { a2880e5100 2025-04-19 63: tg_cloned.get_me().await a2880e5100 2025-04-19 64: })?, a2880e5100 2025-04-19 65: owner_chat: UserId(owner), 66: pool: PgPoolOptions::new() 67: .max_connections(5) 68: .acquire_timeout(std::time::Duration::new(300, 0)) 69: .idle_timeout(std::time::Duration::new(60, 0)) 70: .connect_lazy(&settings.get_string("pg")?)?, 71: sources: Arc::new(Mutex::new(HashSet::new())), 72: http_client, 73: }); a2880e5100 2025-04-19 74: /* let clone = core.clone(); 75: task::spawn(async move { 76: loop { 77: let delay = match &clone.autofetch().await { 78: Err(err) => { a2880e5100 2025-04-19 79: if let Err(err) = clone.send(format!("š {:?}", err), None, None).await { a2880e5100 2025-04-19 80: eprintln!("Autofetch error: {}", err); 81: }; 82: std::time::Duration::from_secs(60) 83: }, 84: Ok(time) => *time, 85: }; 86: task::sleep(delay).await; 87: } a2880e5100 2025-04-19 88: }); */ 89: Ok(core) 90: } 91: a2880e5100 2025-04-19 92: pub fn stream(&self) -> Result<()> { a2880e5100 2025-04-19 93: let mut last_update: Option<i32> = None; 94: loop { a2880e5100 2025-04-19 95: let updates = self.tg.get_updates(last_update, None, 300, Some(vec!["message"])); a2880e5100 2025-04-19 96: } 97: Ok(()) 98: } 99: a2880e5100 2025-04-19 100: /* a2880e5100 2025-04-19 101: pub async fn send<'a, S>(&self, msg: S, target: Option<telegram_bot::UserId>, mode: Option<telegram_bot::types::ParseMode>) -> Result<()> a2880e5100 2025-04-19 102: where S: Into<Cow<'a, str>> { a2880e5100 2025-04-19 103: let mode = mode.unwrap_or(telegram_bot::types::ParseMode::Html); a2880e5100 2025-04-19 104: let target = target.unwrap_or(self.owner_chat); a2880e5100 2025-04-19 105: self.request(telegram_bot::SendMessage::new(target, msg).parse_mode(mode)).await?; a2880e5100 2025-04-19 106: Ok(()) a2880e5100 2025-04-19 107: } */ a2880e5100 2025-04-19 108: a2880e5100 2025-04-19 109: /* pub async fn request<Req: telegram_bot::Request> (&self, req: Req) -> Result<<Req::Response as telegram_bot::ResponseType>::Type, RssError> { a2880e5100 2025-04-19 110: loop { a2880e5100 2025-04-19 111: let res = self.tg.send(&req).await; a2880e5100 2025-04-19 112: match res { a2880e5100 2025-04-19 113: Ok(_) => return Ok(res?), a2880e5100 2025-04-19 114: Err(err) => { a2880e5100 2025-04-19 115: match &err { a2880e5100 2025-04-19 116: TgError::Raw(TgrError::TelegramError { description: _, parameters: Some(params) }) => { a2880e5100 2025-04-19 117: if let Some(delay) = params.retry_after { a2880e5100 2025-04-19 118: println!("Throttled, waiting {} senconds.", delay); a2880e5100 2025-04-19 119: task::sleep(std::time::Duration::from_secs(delay.try_into()?)).await; a2880e5100 2025-04-19 120: } else { a2880e5100 2025-04-19 121: return Err(err.into()); a2880e5100 2025-04-19 122: } a2880e5100 2025-04-19 123: }, a2880e5100 2025-04-19 124: _ => return Err(err.into()), a2880e5100 2025-04-19 125: } a2880e5100 2025-04-19 126: }, a2880e5100 2025-04-19 127: }; a2880e5100 2025-04-19 128: } a2880e5100 2025-04-19 129: } */ a2880e5100 2025-04-19 130: a2880e5100 2025-04-19 131: /* pub async fn check<S>(&self, id: &i32, owner: S, real: bool) -> Result<Cow<'_, str>> a2880e5100 2025-04-19 132: where S: Into<i64> { a2880e5100 2025-04-19 133: let owner = owner.into(); 134: let mut posted: i32 = 0; 135: let mut conn = self.pool.acquire().await?; 136: 137: let id = { 138: let mut set = self.sources.lock().unwrap(); 139: match set.get(id) { 140: Some(id) => id.clone(), 141: None => { 142: let id = Arc::new(*id); 143: set.insert(id.clone()); 144: id.clone() 145: }, 146: } 147: }; 148: let count = Arc::strong_count(&id); 149: if count == 2 { 150: let source = sqlx::query!("select source_id, channel_id, url, iv_hash, owner, url_re from rsstg_source where source_id = $1 and owner = $2", 151: *id, owner).fetch_one(&mut *conn).await?; 152: let destination = match real { a2880e5100 2025-04-19 153: true => telegram_bot::UserId::new(source.channel_id), a2880e5100 2025-04-19 154: false => telegram_bot::UserId::new(source.owner), 155: }; 156: let mut this_fetch: Option<DateTime<chrono::FixedOffset>> = None; 157: let mut posts: BTreeMap<DateTime<chrono::FixedOffset>, String> = BTreeMap::new(); 158: 159: let response = self.http_client.get(&source.url).send().await?; 160: let status = response.status(); 161: let content = response.bytes().await?; 162: match rss::Channel::read_from(&content[..]) { 163: Ok(feed) => { 164: for item in feed.items() { 165: if let Some(link) = item.link() { 166: let date = match item.pub_date() { 167: Some(feed_date) => DateTime::parse_from_rfc2822(feed_date), 168: None => DateTime::parse_from_rfc3339(&item.dublin_core_ext().unwrap().dates()[0]), 169: }?; 170: let url = link; 171: posts.insert(date, url.to_string()); 172: } 173: }; 174: }, 175: Err(err) => match err { 176: rss::Error::InvalidStartTag => { 177: match atom_syndication::Feed::read_from(&content[..]) { 178: Ok(feed) => { 179: for item in feed.entries() { 180: let date = item.published().unwrap(); 181: let url = item.links()[0].href(); 182: posts.insert(*date, url.to_string()); 183: }; 184: }, 185: Err(err) => { a2880e5100 2025-04-19 186: bail!("Unsupported or mangled content:\n{:?}\n{:#?}\n{:#?}\n", &source.url, err, status) 187: }, 188: } 189: }, 190: rss::Error::Eof => (), a2880e5100 2025-04-19 191: _ => bail!("Unsupported or mangled content:\n{:?}\n{:#?}\n{:#?}\n", &source.url, err, status) 192: } 193: }; 194: for (date, url) in posts.iter() { 195: let post_url: Cow<str> = match source.url_re { 196: Some(ref x) => sedregex::ReplaceCommand::new(x)?.execute(url), 197: None => url.into(), 198: }; 199: if let Some(exists) = sqlx::query!("select exists(select true from rsstg_post where url = $1 and source_id = $2) as exists;", 200: &post_url, *id).fetch_one(&mut *conn).await?.exists { 201: if ! exists { 202: if this_fetch.is_none() || *date > this_fetch.unwrap() { 203: this_fetch = Some(*date); 204: }; a2880e5100 2025-04-19 205: self.request( match &source.iv_hash { a2880e5100 2025-04-19 206: Some(hash) => telegram_bot::SendMessage::new(destination, format!("<a href=\"https://t.me/iv?url={}&rhash={}\"> </a>{0}", &post_url, hash)), a2880e5100 2025-04-19 207: None => telegram_bot::SendMessage::new(destination, format!("{}", post_url)), a2880e5100 2025-04-19 208: }.parse_mode(telegram_bot::types::ParseMode::Html)).await a2880e5100 2025-04-19 209: .context("Can't post message:")?; 210: sqlx::query!("insert into rsstg_post (source_id, posted, url) values ($1, $2, $3);", 211: *id, date, &post_url).execute(&mut *conn).await?; 212: }; 213: }; 214: posted += 1; 215: }; 216: posts.clear(); 217: }; 218: sqlx::query!("update rsstg_source set last_scrape = now() where source_id = $1;", 219: *id).execute(&mut *conn).await?; a2880e5100 2025-04-19 220: Ok(format!("Posted: {}", &posted).into()) a2880e5100 2025-04-19 221: } */ 222: a2880e5100 2025-04-19 223: /* pub async fn delete<S>(&self, source_id: &i32, owner: S) -> Result<Cow<'_, str>> a2880e5100 2025-04-19 224: where S: Into<i64> { a2880e5100 2025-04-19 225: let owner = owner.into(); a2880e5100 2025-04-19 226: 227: match sqlx::query!("delete from rsstg_source where source_id = $1 and owner = $2;", 228: source_id, owner).execute(&mut *self.pool.acquire().await?).await?.rows_affected() { 229: 0 => { Ok("No data found found.".into()) }, 230: x => { Ok(format!("{} sources removed.", x).into()) }, 231: } a2880e5100 2025-04-19 232: } */ 233: a2880e5100 2025-04-19 234: /* pub async fn clean<S>(&self, source_id: &i32, owner: S) -> Result<Cow<'_, str>> a2880e5100 2025-04-19 235: where S: Into<i64> { a2880e5100 2025-04-19 236: let owner = owner.into(); a2880e5100 2025-04-19 237: 238: match sqlx::query!("delete from rsstg_post p using rsstg_source s where p.source_id = $1 and owner = $2 and p.source_id = s.source_id;", 239: source_id, owner).execute(&mut *self.pool.acquire().await?).await?.rows_affected() { 240: 0 => { Ok("No data found found.".into()) }, a2880e5100 2025-04-19 241: x => { Ok(format!("{} posts purged.", x).into()) }, 242: } a2880e5100 2025-04-19 243: } */ 244: a2880e5100 2025-04-19 245: /* pub async fn enable<S>(&self, source_id: &i32, owner: S) -> Result<&str> a2880e5100 2025-04-19 246: where S: Into<i64> { a2880e5100 2025-04-19 247: let owner = owner.into(); a2880e5100 2025-04-19 248: 249: match sqlx::query!("update rsstg_source set enabled = true where source_id = $1 and owner = $2", 250: source_id, owner).execute(&mut *self.pool.acquire().await?).await?.rows_affected() { 251: 1 => { Ok("Source enabled.") }, 252: 0 => { Ok("Source not found.") }, 253: _ => { Err(anyhow!("Database error.")) }, 254: } a2880e5100 2025-04-19 255: } */ 256: a2880e5100 2025-04-19 257: /* pub async fn disable<S>(&self, source_id: &i32, owner: S) -> Result<&str> a2880e5100 2025-04-19 258: where S: Into<i64> { a2880e5100 2025-04-19 259: let owner = owner.into(); a2880e5100 2025-04-19 260: 261: match sqlx::query!("update rsstg_source set enabled = false where source_id = $1 and owner = $2", 262: source_id, owner).execute(&mut *self.pool.acquire().await?).await?.rows_affected() { 263: 1 => { Ok("Source disabled.") }, 264: 0 => { Ok("Source not found.") }, 265: _ => { Err(anyhow!("Database error.")) }, 266: } a2880e5100 2025-04-19 267: } */ 268: a2880e5100 2025-04-19 269: /* pub async fn update<S>(&self, update: Option<i32>, channel: &str, channel_id: i64, url: &str, iv_hash: Option<&str>, url_re: Option<&str>, owner: S) -> Result<&str> a2880e5100 2025-04-19 270: where S: Into<i64> { a2880e5100 2025-04-19 271: let owner = owner.into(); 272: let mut conn = self.pool.acquire().await?; 273: 274: match match update { 275: Some(id) => { 276: sqlx::query!("update rsstg_source set channel_id = $2, url = $3, iv_hash = $4, owner = $5, channel = $6, url_re = $7 where source_id = $1", 277: id, channel_id, url, iv_hash, owner, channel, url_re).execute(&mut *conn).await 278: }, 279: None => { 280: sqlx::query!("insert into rsstg_source (channel_id, url, iv_hash, owner, channel, url_re) values ($1, $2, $3, $4, $5, $6)", 281: channel_id, url, iv_hash, owner, channel, url_re).execute(&mut *conn).await 282: }, 283: } { 284: Ok(_) => Ok(match update { 285: Some(_) => "Channel updated.", 286: None => "Channel added.", 287: }), 288: Err(sqlx::Error::Database(err)) => { 289: match err.downcast::<sqlx::postgres::PgDatabaseError>().routine() { 290: Some("_bt_check_unique", ) => { 291: Ok("Duplicate key.") 292: }, 293: Some(_) => { 294: Ok("Database error.") 295: }, 296: None => { 297: Ok("No database error extracted.") 298: }, 299: } 300: }, 301: Err(err) => { a2880e5100 2025-04-19 302: bail!("Sorry, unknown error:\n{:#?}\n", err); 303: }, 304: } 305: } 306: 307: async fn autofetch(&self) -> Result<std::time::Duration> { 308: let mut delay = chrono::Duration::minutes(1); 309: let now = chrono::Local::now(); 310: let mut queue = sqlx::query!(r#"select source_id, next_fetch as "next_fetch: DateTime<chrono::Local>", owner from rsstg_order natural left join rsstg_source where next_fetch < now() + interval '1 minute';"#) 311: .fetch_all(&mut *self.pool.acquire().await?).await?; 312: for row in queue.iter() { 313: if let Some(next_fetch) = row.next_fetch { 314: if next_fetch < now { 315: if let (Some(owner), Some(source_id)) = (row.owner, row.source_id) { 316: let clone = Core { a2880e5100 2025-04-19 317: owner_chat: telegram_bot::UserId::new(owner), 318: ..self.clone() 319: }; 320: task::spawn(async move { 321: if let Err(err) = clone.check(&source_id, owner, true).await { a2880e5100 2025-04-19 322: if let Err(err) = clone.send(&format!("š {:?}", err), None, None).await { a2880e5100 2025-04-19 323: dbg!("Check error: {}", err); 324: // clone.disable(&source_id, owner).await.unwrap(); 325: }; 326: }; 327: }); 328: } 329: } else if next_fetch - now < delay { 330: delay = next_fetch - now; 331: } 332: } 333: }; 334: queue.clear(); 335: Ok(delay.to_std()?) 336: } 337: a2880e5100 2025-04-19 338: pub async fn list<S>(&self, owner: S) -> Result<String> a2880e5100 2025-04-19 339: where S: Into<i64> { a2880e5100 2025-04-19 340: let owner = owner.into(); a2880e5100 2025-04-19 341: 342: let mut reply: Vec<Cow<str>> = vec![]; 343: reply.push("Channels:".into()); 344: let rows = sqlx::query!("select source_id, channel, enabled, url, iv_hash, url_re from rsstg_source where owner = $1 order by source_id", 345: owner).fetch_all(&mut *self.pool.acquire().await?).await?; 346: for row in rows.iter() { 347: reply.push(format!("\n\\#ļøā£ {} \\*ļøā£ `{}` {}\nš `{}`", row.source_id, row.channel, 348: match row.enabled { 349: true => "š enabled", 350: false => "ā disabled", 351: }, row.url).into()); 352: if let Some(hash) = &row.iv_hash { a2880e5100 2025-04-19 353: reply.push(format!("IV: `{}`", hash).into()); 354: } 355: if let Some(re) = &row.url_re { a2880e5100 2025-04-19 356: reply.push(format!("RE: `{}`", re).into()); 357: } 358: }; 359: Ok(reply.join("\n")) a2880e5100 2025-04-19 360: } */ 361: }