1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
|
use std::collections::{BTreeMap, HashSet};
use std::sync::{Arc, Mutex};
use config;
use tokio;
use reqwest;
use rss;
use atom_syndication;
use chrono::DateTime;
use regex::Regex;
use telegram_bot::*;
use tokio::stream::StreamExt;
use sqlx::postgres::PgPoolOptions;
use sqlx::Row;
#[macro_use]
extern crate lazy_static;
use anyhow::{anyhow, bail, Context, Result};
#[derive(Clone)]
|
>
|
|
>
>
>
<
<
<
<
|
<
<
<
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
|
use std::collections::{BTreeMap, HashSet};
use std::sync::{Arc, Mutex};
use chrono::DateTime;
use config;
use futures::StreamExt;
use regex::Regex;
use reqwest;
use sqlx::postgres::PgPoolOptions;
use sqlx::Row;
use tokio;
use rss;
use atom_syndication;
use telegram_bot::*;
//use tokio::stream::StreamExt;
#[macro_use]
extern crate lazy_static;
use anyhow::{anyhow, bail, Context, Result};
#[derive(Clone)]
|
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
|
sqlx::query("insert into rsstg_post (source_id, posted, url) values ($1, $2, $3);")
.bind(*id)
.bind(date)
.bind(url)
.execute(&mut conn).await
.with_context(|| format!("Record post:\n{:?}", &conn))?;
drop(conn);
tokio::time::delay_for(std::time::Duration::new(4, 0)).await;
};
};
posts.clear();
};
let mut conn = self.pool.acquire().await
.with_context(|| format!("Update scrape fetch conn:\n{:?}", &self.pool))?;
sqlx::query("update rsstg_source set last_scrape = now() where source_id = $1;")
|
|
|
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
|
sqlx::query("insert into rsstg_post (source_id, posted, url) values ($1, $2, $3);")
.bind(*id)
.bind(date)
.bind(url)
.execute(&mut conn).await
.with_context(|| format!("Record post:\n{:?}", &conn))?;
drop(conn);
tokio::time::sleep(std::time::Duration::new(4, 0)).await;
};
};
posts.clear();
};
let mut conn = self.pool.acquire().await
.with_context(|| format!("Update scrape fetch conn:\n{:?}", &self.pool))?;
sqlx::query("update rsstg_source set last_scrape = now() where source_id = $1;")
|
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
|
} else {
if next_fetch - now < delay {
delay = next_fetch - now;
}
}
};
queue.clear();
tokio::time::delay_for(delay.to_std()?).await;
delay = chrono::Duration::minutes(1);
}
}
async fn list<S>(&self, owner: S) -> Result<Vec<String>>
where S: Into<i64> {
let owner = owner.into();
|
|
|
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
|
} else {
if next_fetch - now < delay {
delay = next_fetch - now;
}
}
};
queue.clear();
tokio::time::sleep(delay.to_std()?).await;
delay = chrono::Duration::minutes(1);
}
}
async fn list<S>(&self, owner: S) -> Result<Vec<String>>
where S: Into<i64> {
let owner = owner.into();
|