Check-in [61899b5618]
Logged in as anonymous
Overview
Comment:also mark source_id scraped just after the scrape starts to delay next scrapes in case of error
Downloads: Tarball | ZIP archive | SQL archive
Timelines: family | ancestors | descendants | both | trunk | v0.3.2
Files: files | file ages | folders
SHA3-256: 61899b5618a77aeb927331e36c8fe1d1407940695bb6943e83c185b99712f896
User & Date: arcade on 2025-04-26 06:32:25.625
Other Links: manifest | tags
Context
2025-05-20
09:04
add some error handling, throttling check-in: 1bd041d00f user: arcade tags: trunk, v0.3.3
2025-04-26
06:32
also mark source_id scraped just after the scrape starts to delay next scrapes in case of error check-in: 61899b5618 user: arcade tags: trunk, v0.3.2
06:10
fix int size, small bump check-in: e3f7eeb26a user: arcade tags: trunk, v0.3.2
Changes
2151
2152
2153
2154
2155
2156
2157
2158

2159
2160
2161
2162
2163
2164
2165
2151
2152
2153
2154
2155
2156
2157

2158
2159
2160
2161
2162
2163
2164
2165







-
+







 "derive_builder",
 "never",
 "quick-xml",
]

[[package]]
name = "rsstg"
version = "0.3.1"
version = "0.3.2"
dependencies = [
 "anyhow",
 "async-std",
 "atom_syndication",
 "chrono",
 "config",
 "frankenstein",
176
177
178
179
180
181
182

183
184
185
186
187
188
189
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190







+







					id.clone()
				},
			}
		};
		let count = Arc::strong_count(&id);
		if count == 2 {
			let source = conn.get_source(*id, owner).await?;
			conn.set_scrape(*id).await?;
			let destination = match real {
				true => source.channel_id,
				false => source.owner,
			};
			let mut this_fetch: Option<DateTime<chrono::FixedOffset>> = None;
			let mut posts: BTreeMap<DateTime<chrono::FixedOffset>, String> = BTreeMap::new();

239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
240
241
242
243
244
245
246

247
248
249
250
251
252
253







-







						conn.add_post(*id, date, &post_url).await?;
					};
				};
				posted += 1;
			};
			posts.clear();
		};
		conn.set_scrape(*id).await?;
		Ok(format!("Posted: {posted}"))
	}

	async fn autofetch(&mut self) -> Result<std::time::Duration> {
		let mut delay = chrono::Duration::minutes(1);
		let now = chrono::Local::now();
		let mut conn = self.db.begin().await?;