227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
|
let mut posts: BTreeMap<DateTime<chrono::FixedOffset>, Post> = BTreeMap::new();
let mut builder = self.http_client.get(&source.url);
if let Some(last_scrape) = last_scrape {
builder = builder.header(LAST_MODIFIED, last_scrape.to_rfc2822());
};
let response = builder.send().await.stack()?;
#[cfg(debug_assertions)]
{
let headers = response.headers();
let expires = headers.get(EXPIRES);
let cache = headers.get(CACHE_CONTROL);
if expires.is_some() || cache.is_some() {
println!("{} {} {:?} {:?} {:?}", Local::now().to_rfc2822(), source.url, last_scrape, expires, cache);
}
}
let status = response.status();
let content = response.bytes().await.stack()?;
match rss::Channel::read_from(&content[..]) {
Ok(feed) => {
for item in feed.items() {
if let Some(link) = item.link() {
let date = match item.pub_date() {
|
>
>
>
|
>
>
>
>
|
|
|
|
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
|
let mut posts: BTreeMap<DateTime<chrono::FixedOffset>, Post> = BTreeMap::new();
let mut builder = self.http_client.get(&source.url);
if let Some(last_scrape) = last_scrape {
builder = builder.header(LAST_MODIFIED, last_scrape.to_rfc2822());
};
let response = builder.send().await.stack()?;
// Print debug output for each response processed to track how expires
// are handled
#[cfg(debug_assertions)] {
use reqwest::header::{
EXPIRES,
CACHE_CONTROL,
};
let headers = response.headers();
let expires = headers.get(EXPIRES);
let cache = headers.get(CACHE_CONTROL);
if expires.is_some() || cache.is_some() {
println!("{} {} {:?} {:?} {:?}", Local::now().to_rfc2822(), source.url, last_scrape, expires, cache);
} }
let status = response.status();
let content = response.bytes().await.stack()?;
match rss::Channel::read_from(&content[..]) {
Ok(feed) => {
for item in feed.items() {
if let Some(link) = item.link() {
let date = match item.pub_date() {
|