From 4db65c191c8b9198913c12f192fb641bbb709c08 Mon Sep 17 00:00:00 2001 From: Vijay Ramesh Date: Mon, 19 Jun 2023 23:17:54 -0700 Subject: [PATCH 01/45] after 30 days post deletion, replace comment.content and post.body with 'Permanently Deleted' --- crates/db_schema/src/impls/comment.rs | 5 ++- crates/db_schema/src/impls/post.rs | 18 +++++---- crates/db_schema/src/utils.rs | 3 ++ src/scheduled_tasks.rs | 55 ++++++++++++++++++++++++++- 4 files changed, 71 insertions(+), 10 deletions(-) diff --git a/crates/db_schema/src/impls/comment.rs b/crates/db_schema/src/impls/comment.rs index 46045cd10..8aa8c1793 100644 --- a/crates/db_schema/src/impls/comment.rs +++ b/crates/db_schema/src/impls/comment.rs @@ -11,7 +11,7 @@ use crate::{ CommentUpdateForm, }, traits::{Crud, Likeable, Saveable}, - utils::{get_conn, naive_now, DbPool}, + utils::{get_conn, naive_now, DbPool, DELETED_REPLACEMENT_TEXT}, }; use diesel::{ dsl::{insert_into, sql_query}, @@ -29,9 +29,10 @@ impl Comment { for_creator_id: PersonId, ) -> Result, Error> { let conn = &mut get_conn(pool).await?; + diesel::update(comment.filter(creator_id.eq(for_creator_id))) .set(( - content.eq("*Permananently Deleted*"), + content.eq(DELETED_REPLACEMENT_TEXT), deleted.eq(true), updated.eq(naive_now()), )) diff --git a/crates/db_schema/src/impls/post.rs b/crates/db_schema/src/impls/post.rs index a3309428c..7f59d29ec 100644 --- a/crates/db_schema/src/impls/post.rs +++ b/crates/db_schema/src/impls/post.rs @@ -27,7 +27,14 @@ use crate::{ PostUpdateForm, }, traits::{Crud, Likeable, Readable, Saveable}, - utils::{get_conn, naive_now, DbPool, FETCH_LIMIT_MAX}, + utils::{ + get_conn, + naive_now, + DbPool, + DELETED_REPLACEMENT_TEXT, + DELETED_REPLACEMENT_URL, + FETCH_LIMIT_MAX, + }, }; use ::url::Url; use diesel::{dsl::insert_into, result::Error, ExpressionMethods, QueryDsl, TextExpressionMethods}; @@ -111,14 +118,11 @@ impl Post { ) -> Result, Error> { let conn = &mut get_conn(pool).await?; - let perma_deleted = "*Permananently Deleted*"; - let perma_deleted_url = "https://deleted.com"; - diesel::update(post.filter(creator_id.eq(for_creator_id))) .set(( - name.eq(perma_deleted), - url.eq(perma_deleted_url), - body.eq(perma_deleted), + name.eq(DELETED_REPLACEMENT_TEXT), + url.eq(DELETED_REPLACEMENT_URL), + body.eq(DELETED_REPLACEMENT_TEXT), deleted.eq(true), updated.eq(naive_now()), )) diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index 98d3952ab..9954f623b 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -222,6 +222,9 @@ pub mod functions { sql_function!(fn lower(x: Text) -> Text); } +pub const DELETED_REPLACEMENT_TEXT: &str = "*Permanently Deleted*"; +pub const DELETED_REPLACEMENT_URL: &str = "https://join-lemmy.org/"; + impl ToSql for DbUrl { fn to_sql(&self, out: &mut Output) -> diesel::serialize::Result { >::to_sql(&self.0.to_string(), &mut out.reborrow()) diff --git a/src/scheduled_tasks.rs b/src/scheduled_tasks.rs index 9fb1ba702..891dca365 100644 --- a/src/scheduled_tasks.rs +++ b/src/scheduled_tasks.rs @@ -3,6 +3,7 @@ use diesel::{ dsl::{now, IntervalDsl}, Connection, ExpressionMethods, + NullableExpressionMethods, QueryDsl, }; // Import week days and WeekDay @@ -11,15 +12,17 @@ use lemmy_api_common::context::LemmyContext; use lemmy_db_schema::{ schema::{ activity, + comment, comment_aggregates, community_aggregates, community_person_ban, instance, person, + post, post_aggregates, }, source::instance::{Instance, InstanceForm}, - utils::{functions::hot_rank, naive_now}, + utils::{functions::hot_rank, naive_now, DELETED_REPLACEMENT_TEXT}, }; use lemmy_routes::nodeinfo::NodeInfo; use lemmy_utils::{error::LemmyError, REQWEST_TIMEOUT}; @@ -66,6 +69,13 @@ pub fn setup( context_1.settings_updated_channel().remove_older_than(hour); }); + // Overwrite deleted & removed posts and comments every day + let url = db_url.clone(); + scheduler.every(CTimeUnits::days(1)).run(move || { + let mut conn = PgConnection::establish(&url).expect("could not establish connection"); + overwrite_deleted_posts_and_comments(&mut conn); + }); + // Update the Instance Software scheduler.every(CTimeUnits::days(1)).run(move || { let mut conn = PgConnection::establish(&db_url).expect("could not establish connection"); @@ -86,6 +96,7 @@ fn startup_jobs(db_url: &str) { update_hot_ranks(&mut conn, false); update_banned_when_expired(&mut conn); clear_old_activities(&mut conn); + overwrite_deleted_posts_and_comments(&mut conn); } /// Update the hot_rank columns for the aggregates tables @@ -166,6 +177,48 @@ fn clear_old_activities(conn: &mut PgConnection) { } } +/// overwrite posts and comments 30d after deletion +fn overwrite_deleted_posts_and_comments(conn: &mut PgConnection) { + info!("Overwriting deleted posts..."); + match diesel::update( + post::table + .filter(post::deleted.eq(true)) + .filter(post::updated.lt(now.nullable() - 1.months())) + .filter(post::body.ne(DELETED_REPLACEMENT_TEXT)), + ) + .set(( + post::body.eq(DELETED_REPLACEMENT_TEXT), + post::name.eq(DELETED_REPLACEMENT_TEXT), + )) + .execute(conn) + { + Ok(_) => { + info!("Done."); + } + Err(e) => { + error!("Failed to overwrite deleted posts: {}", e) + } + } + + info!("Overwriting deleted comments..."); + match diesel::update( + comment::table + .filter(comment::deleted.eq(true)) + .filter(comment::updated.lt(now.nullable() - 1.months())) + .filter(comment::content.ne(DELETED_REPLACEMENT_TEXT)), + ) + .set(comment::content.eq(DELETED_REPLACEMENT_TEXT)) + .execute(conn) + { + Ok(_) => { + info!("Done."); + } + Err(e) => { + error!("Failed to overwrite deleted comments: {}", e) + } + } +} + /// Re-calculate the site and community active counts every 12 hours fn active_counts(conn: &mut PgConnection) { info!("Updating active site and community aggregates ..."); From d7da911a480756dc6607ecce86516205ed81e99c Mon Sep 17 00:00:00 2001 From: cetra3 Date: Mon, 26 Jun 2023 17:54:11 +0930 Subject: [PATCH 02/45] Remove `actix_rt` & use standard tokio spawn (#3158) * Remove `actix_rt` & use standard tokio spawn * Adjust rust log back down * Format correctly * Update cargo lock * Add DB settings * Change name and update to latest rev * Clean up formatting changes * Move `worker_count` and `worker_retry_count` to settings * Update defaults * Use `0.4.4` instead of git branch --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- config/defaults.hjson | 4 ++++ crates/api_common/Cargo.toml | 2 +- crates/api_common/src/request.rs | 2 +- crates/api_common/src/site.rs | 3 --- crates/api_crud/src/site/create.rs | 1 - crates/api_crud/src/site/update.rs | 1 - crates/apub/Cargo.toml | 2 +- crates/apub/src/collections/community_moderators.rs | 2 +- crates/apub/src/objects/comment.rs | 6 +++--- crates/apub/src/objects/community.rs | 2 +- crates/apub/src/objects/instance.rs | 2 +- crates/apub/src/objects/person.rs | 4 ++-- crates/apub/src/objects/post.rs | 2 +- crates/apub/src/objects/private_message.rs | 4 ++-- crates/db_schema/src/schema.rs | 1 - crates/db_schema/src/source/local_site.rs | 4 ---- crates/utils/src/settings/structs.rs | 6 ++++++ .../down.sql | 1 + .../up.sql | 1 + src/lib.rs | 13 ++++++------- src/main.rs | 2 +- 23 files changed, 36 insertions(+), 35 deletions(-) create mode 100644 migrations/2023-06-19-055530_add_retry_worker_setting/down.sql create mode 100644 migrations/2023-06-19-055530_add_retry_worker_setting/up.sql diff --git a/Cargo.lock b/Cargo.lock index 9590e8f13..9d575f578 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2543,7 +2543,6 @@ dependencies = [ name = "lemmy_api_common" version = "0.18.0" dependencies = [ - "actix-rt", "actix-web", "anyhow", "chrono", @@ -2561,6 +2560,7 @@ dependencies = [ "rosetta-i18n", "serde", "serde_with", + "tokio", "tracing", "ts-rs", "url", @@ -2592,7 +2592,6 @@ name = "lemmy_apub" version = "0.18.0" dependencies = [ "activitypub_federation", - "actix-rt", "actix-web", "anyhow", "assert-json-diff", @@ -2620,6 +2619,7 @@ dependencies = [ "sha2", "strum_macros", "task-local-extensions", + "tokio", "tracing", "url", "uuid", diff --git a/Cargo.toml b/Cargo.toml index 430deb082..07e41ab3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,7 +89,7 @@ anyhow = "1.0.71" diesel_ltree = "0.3.0" typed-builder = "0.10.0" serial_test = "0.9.0" -tokio = "1.28.2" +tokio = { version = "1.28.2", features = ["full"] } sha2 = "0.10.6" regex = "1.8.4" once_cell = "1.18.0" diff --git a/config/defaults.hjson b/config/defaults.hjson index 4c38ddd45..6032f8fc9 100644 --- a/config/defaults.hjson +++ b/config/defaults.hjson @@ -76,4 +76,8 @@ port: 8536 # Whether the site is available over TLS. Needs to be true for federation to work. tls_enabled: true + # The number of activitypub federation workers that can be in-flight concurrently + worker_count: 0 + # The number of activitypub federation retry workers that can be in-flight concurrently + retry_count: 0 } diff --git a/crates/api_common/Cargo.toml b/crates/api_common/Cargo.toml index 46045d805..339d233a1 100644 --- a/crates/api_common/Cargo.toml +++ b/crates/api_common/Cargo.toml @@ -38,7 +38,7 @@ encoding = { version = "0.2.33", optional = true } anyhow = { workspace = true } futures = { workspace = true } uuid = { workspace = true } -actix-rt = { workspace = true } +tokio = { workspace = true } reqwest = { workspace = true } ts-rs = { workspace = true, optional = true } actix-web = { workspace = true } diff --git a/crates/api_common/src/request.rs b/crates/api_common/src/request.rs index c6f71b868..3139193a6 100644 --- a/crates/api_common/src/request.rs +++ b/crates/api_common/src/request.rs @@ -271,7 +271,7 @@ mod tests { use url::Url; // These helped with testing - #[actix_rt::test] + #[tokio::test] async fn test_site_metadata() { let settings = &SETTINGS.clone(); let client = reqwest::Client::builder() diff --git a/crates/api_common/src/site.rs b/crates/api_common/src/site.rs index 4d488ec1b..865acc0dc 100644 --- a/crates/api_common/src/site.rs +++ b/crates/api_common/src/site.rs @@ -177,7 +177,6 @@ pub struct CreateSite { pub rate_limit_search_per_second: Option, pub federation_enabled: Option, pub federation_debug: Option, - pub federation_worker_count: Option, pub captcha_enabled: Option, pub captcha_difficulty: Option, pub allowed_instances: Option>, @@ -250,8 +249,6 @@ pub struct EditSite { pub federation_enabled: Option, /// Enables federation debugging. pub federation_debug: Option, - /// The number of federation workers. - pub federation_worker_count: Option, /// Whether to enable captchas for signups. pub captcha_enabled: Option, /// The captcha difficulty. Can be easy, medium, or hard diff --git a/crates/api_crud/src/site/create.rs b/crates/api_crud/src/site/create.rs index 2a51309a4..a1669baef 100644 --- a/crates/api_crud/src/site/create.rs +++ b/crates/api_crud/src/site/create.rs @@ -122,7 +122,6 @@ impl PerformCrud for CreateSite { .slur_filter_regex(diesel_option_overwrite(&data.slur_filter_regex)) .actor_name_max_length(data.actor_name_max_length) .federation_enabled(data.federation_enabled) - .federation_worker_count(data.federation_worker_count) .captcha_enabled(data.captcha_enabled) .captcha_difficulty(data.captcha_difficulty.clone()) .build(); diff --git a/crates/api_crud/src/site/update.rs b/crates/api_crud/src/site/update.rs index fadde0a0b..6664d549a 100644 --- a/crates/api_crud/src/site/update.rs +++ b/crates/api_crud/src/site/update.rs @@ -123,7 +123,6 @@ impl PerformCrud for EditSite { .slur_filter_regex(diesel_option_overwrite(&data.slur_filter_regex)) .actor_name_max_length(data.actor_name_max_length) .federation_enabled(data.federation_enabled) - .federation_worker_count(data.federation_worker_count) .captcha_enabled(data.captcha_enabled) .captcha_difficulty(data.captcha_difficulty.clone()) .reports_email_admins(data.reports_email_admins) diff --git a/crates/apub/Cargo.toml b/crates/apub/Cargo.toml index 2007b541a..8570541f7 100644 --- a/crates/apub/Cargo.toml +++ b/crates/apub/Cargo.toml @@ -25,7 +25,7 @@ chrono = { workspace = true } serde_json = { workspace = true } serde = { workspace = true } actix-web = { workspace = true } -actix-rt = { workspace = true } +tokio = {workspace = true} tracing = { workspace = true } strum_macros = { workspace = true } url = { workspace = true } diff --git a/crates/apub/src/collections/community_moderators.rs b/crates/apub/src/collections/community_moderators.rs index c439da710..d53f86280 100644 --- a/crates/apub/src/collections/community_moderators.rs +++ b/crates/apub/src/collections/community_moderators.rs @@ -120,7 +120,7 @@ mod tests { }; use serial_test::serial; - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_community_moderators() { let context = init_context().await; diff --git a/crates/apub/src/objects/comment.rs b/crates/apub/src/objects/comment.rs index e2a03b8b3..16cb1542b 100644 --- a/crates/apub/src/objects/comment.rs +++ b/crates/apub/src/objects/comment.rs @@ -223,7 +223,7 @@ pub(crate) mod tests { LocalSite::delete(context.pool()).await.unwrap(); } - #[actix_rt::test] + #[tokio::test] #[serial] pub(crate) async fn test_parse_lemmy_comment() { let context = init_context().await; @@ -249,7 +249,7 @@ pub(crate) mod tests { cleanup(data, &context).await; } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_pleroma_comment() { let context = init_context().await; @@ -279,7 +279,7 @@ pub(crate) mod tests { cleanup(data, &context).await; } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_html_to_markdown_sanitize() { let parsed = parse_html("hello"); diff --git a/crates/apub/src/objects/community.rs b/crates/apub/src/objects/community.rs index 6526d2d26..888a7f458 100644 --- a/crates/apub/src/objects/community.rs +++ b/crates/apub/src/objects/community.rs @@ -242,7 +242,7 @@ pub(crate) mod tests { community } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_community() { let context = init_context().await; diff --git a/crates/apub/src/objects/instance.rs b/crates/apub/src/objects/instance.rs index 72d133441..6cd27fbbd 100644 --- a/crates/apub/src/objects/instance.rs +++ b/crates/apub/src/objects/instance.rs @@ -221,7 +221,7 @@ pub(crate) mod tests { site } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_instance() { let context = init_context().await; diff --git a/crates/apub/src/objects/person.rs b/crates/apub/src/objects/person.rs index c71d46ccf..3eeb733fd 100644 --- a/crates/apub/src/objects/person.rs +++ b/crates/apub/src/objects/person.rs @@ -223,7 +223,7 @@ pub(crate) mod tests { (person, site) } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_person() { let context = init_context().await; @@ -236,7 +236,7 @@ pub(crate) mod tests { cleanup((person, site), &context).await; } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_pleroma_person() { let context = init_context().await; diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index b255ffb9b..4ef9351ab 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -281,7 +281,7 @@ mod tests { use lemmy_db_schema::source::site::Site; use serial_test::serial; - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_post() { let context = init_context().await; diff --git a/crates/apub/src/objects/private_message.rs b/crates/apub/src/objects/private_message.rs index 01f576ff8..ae2637c58 100644 --- a/crates/apub/src/objects/private_message.rs +++ b/crates/apub/src/objects/private_message.rs @@ -187,7 +187,7 @@ mod tests { Site::delete(context.pool(), data.2.id).await.unwrap(); } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_pm() { let context = init_context().await; @@ -213,7 +213,7 @@ mod tests { cleanup(data, &context).await; } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_pleroma_pm() { let context = init_context().await; diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index ac4ddc47a..6714913f4 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -339,7 +339,6 @@ diesel::table! { slur_filter_regex -> Nullable, actor_name_max_length -> Int4, federation_enabled -> Bool, - federation_worker_count -> Int4, captcha_enabled -> Bool, #[max_length = 255] captcha_difficulty -> Varchar, diff --git a/crates/db_schema/src/source/local_site.rs b/crates/db_schema/src/source/local_site.rs index e65a61535..be93717a9 100644 --- a/crates/db_schema/src/source/local_site.rs +++ b/crates/db_schema/src/source/local_site.rs @@ -50,8 +50,6 @@ pub struct LocalSite { pub actor_name_max_length: i32, /// Whether federation is enabled. pub federation_enabled: bool, - /// The number of concurrent federation http workers. - pub federation_worker_count: i32, /// Whether captcha is enabled. pub captcha_enabled: bool, /// The captcha difficulty. @@ -85,7 +83,6 @@ pub struct LocalSiteInsertForm { pub slur_filter_regex: Option, pub actor_name_max_length: Option, pub federation_enabled: Option, - pub federation_worker_count: Option, pub captcha_enabled: Option, pub captcha_difficulty: Option, pub registration_mode: Option, @@ -112,7 +109,6 @@ pub struct LocalSiteUpdateForm { pub slur_filter_regex: Option>, pub actor_name_max_length: Option, pub federation_enabled: Option, - pub federation_worker_count: Option, pub captcha_enabled: Option, pub captcha_difficulty: Option, pub registration_mode: Option, diff --git a/crates/utils/src/settings/structs.rs b/crates/utils/src/settings/structs.rs index 6e200b224..5d0e642f6 100644 --- a/crates/utils/src/settings/structs.rs +++ b/crates/utils/src/settings/structs.rs @@ -39,6 +39,12 @@ pub struct Settings { #[default(None)] #[doku(skip)] pub opentelemetry_url: Option, + /// The number of activitypub federation workers that can be in-flight concurrently + #[default(0)] + pub worker_count: usize, + /// The number of activitypub federation retry workers that can be in-flight concurrently + #[default(0)] + pub retry_count: usize, } #[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document)] diff --git a/migrations/2023-06-19-055530_add_retry_worker_setting/down.sql b/migrations/2023-06-19-055530_add_retry_worker_setting/down.sql new file mode 100644 index 000000000..e3c200a15 --- /dev/null +++ b/migrations/2023-06-19-055530_add_retry_worker_setting/down.sql @@ -0,0 +1 @@ +alter table local_site add column federation_worker_count int default 64 not null; \ No newline at end of file diff --git a/migrations/2023-06-19-055530_add_retry_worker_setting/up.sql b/migrations/2023-06-19-055530_add_retry_worker_setting/up.sql new file mode 100644 index 000000000..2aac86f85 --- /dev/null +++ b/migrations/2023-06-19-055530_add_retry_worker_setting/up.sql @@ -0,0 +1 @@ +alter table local_site drop column federation_worker_count; \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 86cf400b6..d919acc05 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -139,24 +139,23 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> { }); } + let settings_bind = settings.clone(); + let federation_config = FederationConfig::builder() .domain(settings.hostname.clone()) .app_data(context.clone()) .client(client.clone()) .http_fetch_limit(FEDERATION_HTTP_FETCH_LIMIT) - .worker_count(local_site.federation_worker_count as usize) + .worker_count(settings.worker_count) + .retry_count(settings.retry_count) .debug(cfg!(debug_assertions)) .http_signature_compat(true) .url_verifier(Box::new(VerifyUrlData(context.pool().clone()))) .build() - .await - .expect("configure federation"); + .await?; // Create Http server with websocket support - let settings_bind = settings.clone(); HttpServer::new(move || { - let context = context.clone(); - let cors_config = if cfg!(debug_assertions) { Cors::permissive() } else { @@ -173,7 +172,7 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> { )) .wrap(cors_config) .wrap(TracingLogger::::new()) - .app_data(Data::new(context)) + .app_data(Data::new(context.clone())) .app_data(Data::new(rate_limit_cell.clone())) .wrap(FederationMiddleware::new(federation_config.clone())) // The routes diff --git a/src/main.rs b/src/main.rs index 315fe84be..5fc03ed02 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ use lemmy_server::{init_logging, start_lemmy_server}; use lemmy_utils::{error::LemmyError, settings::SETTINGS}; -#[actix_web::main] +#[tokio::main] pub async fn main() -> Result<(), LemmyError> { init_logging(&SETTINGS.opentelemetry_url)?; #[cfg(not(feature = "embed-pictrs"))] From 6d67f88603482f62bea74c8ef7ceae4ca4b77d59 Mon Sep 17 00:00:00 2001 From: Sander Saarend Date: Mon, 26 Jun 2023 11:25:38 +0300 Subject: [PATCH 03/45] Add support for sslmode=require for diesel-async DB connections (#3189) --- Cargo.lock | 92 +++++++++++++++++++++++++++++------ Cargo.toml | 9 ++++ crates/db_schema/Cargo.toml | 6 ++- crates/db_schema/src/utils.rs | 64 ++++++++++++++++++++++-- 4 files changed, 152 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d575f578..99d94948f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -215,7 +215,7 @@ dependencies = [ "futures-util", "mio", "num_cpus", - "socket2", + "socket2 0.4.9", "tokio", "tracing", ] @@ -245,7 +245,7 @@ dependencies = [ "http", "log", "pin-project-lite", - "tokio-rustls", + "tokio-rustls 0.23.4", "tokio-util 0.7.4", "webpki-roots", ] @@ -297,7 +297,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2", + "socket2 0.4.9", "time 0.3.15", "url", ] @@ -496,7 +496,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rand 0.8.5", - "rustls", + "rustls 0.20.7", "serde", "serde_json", "serde_urlencoded", @@ -2262,7 +2262,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -2640,9 +2640,11 @@ dependencies = [ "diesel-derive-newtype", "diesel_ltree", "diesel_migrations", + "futures-util", "lemmy_utils", "once_cell", "regex", + "rustls 0.21.2", "serde", "serde_json", "serde_with", @@ -2651,6 +2653,8 @@ dependencies = [ "strum", "strum_macros", "tokio", + "tokio-postgres", + "tokio-postgres-rustls", "tracing", "ts-rs", "typed-builder", @@ -2736,6 +2740,7 @@ dependencies = [ "diesel", "diesel-async", "doku", + "futures-util", "lemmy_api", "lemmy_api_common", "lemmy_api_crud", @@ -2749,9 +2754,12 @@ dependencies = [ "reqwest", "reqwest-middleware", "reqwest-tracing", + "rustls 0.21.2", "serde", "serde_json", "tokio", + "tokio-postgres", + "tokio-postgres-rustls", "tracing", "tracing-actix-web 0.6.2", "tracing-error", @@ -2820,7 +2828,7 @@ dependencies = [ "nom 7.1.1", "once_cell", "quoted_printable", - "socket2", + "socket2 0.4.9", ] [[package]] @@ -3932,11 +3940,11 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "878c6cbf956e03af9aa8204b407b9cbf47c072164800aa918c516cd4b056c50c" +checksum = "78b7fa9f396f51dffd61546fd8573ee20592287996568e6175ceb0f8699ad75d" dependencies = [ - "base64 0.13.1", + "base64 0.21.2", "byteorder", "bytes", "fallible-iterator", @@ -4495,6 +4503,28 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.9" @@ -4859,6 +4889,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "spin" version = "0.5.2" @@ -5304,7 +5344,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.4.9", "tokio-macros", "tracing", "windows-sys 0.48.0", @@ -5343,9 +5383,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29a12c1b3e0704ae7dfc25562629798b29c72e6b1d0a681b6f29ab4ae5e7f7bf" +checksum = "6e89f6234aa8fd43779746012fcf53603cdb91fdd8399aa0de868c2d56b6dde1" dependencies = [ "async-trait", "byteorder", @@ -5360,22 +5400,46 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "socket2", + "socket2 0.5.3", "tokio", "tokio-util 0.7.4", ] +[[package]] +name = "tokio-postgres-rustls" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd5831152cb0d3f79ef5523b357319ba154795d64c7078b2daa95a803b54057f" +dependencies = [ + "futures", + "ring", + "rustls 0.21.2", + "tokio", + "tokio-postgres", + "tokio-rustls 0.24.1", +] + [[package]] name = "tokio-rustls" version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.7", "tokio", "webpki", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.2", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.11" diff --git a/Cargo.toml b/Cargo.toml index 07e41ab3b..2ee5a5308 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,6 +107,10 @@ rand = "0.8.5" opentelemetry = { version = "0.17.0", features = ["rt-tokio"] } tracing-opentelemetry = { version = "0.17.4" } ts-rs = { version = "6.2", features = ["serde-compat", "format", "chrono-impl"] } +rustls = { version ="0.21.2", features = ["dangerous_configuration"]} +futures-util = "0.3.28" +tokio-postgres = "0.7.8" +tokio-postgres-rustls = "0.10.0" [dependencies] lemmy_api = { workspace = true } @@ -140,3 +144,8 @@ opentelemetry-otlp = { version = "0.10.0", optional = true } pict-rs = { version = "0.4.0-rc.3", optional = true } tokio.workspace = true actix-cors = "0.6.4" +rustls = { workspace = true } +futures-util = { workspace = true } +tokio-postgres = { workspace = true } +tokio-postgres-rustls = { workspace = true } + diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index aa26382c0..6c89ab9fa 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -43,7 +43,11 @@ async-trait = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } deadpool = { version = "0.9.5", features = ["rt_tokio_1"], optional = true } -ts-rs = { workspace = true, optional = true } +ts-rs = { workspace = true, optional = true } +rustls = { workspace = true } +futures-util = { workspace = true } +tokio-postgres = { workspace = true } +tokio-postgres-rustls = { workspace = true } [dev-dependencies] serial_test = { workspace = true } diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index 98d3952ab..1319a62f6 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -12,7 +12,7 @@ use diesel::{ backend::Backend, deserialize::FromSql, pg::Pg, - result::{Error as DieselError, Error::QueryBuilderError}, + result::{ConnectionError, ConnectionResult, Error as DieselError, Error::QueryBuilderError}, serialize::{Output, ToSql}, sql_types::Text, PgConnection, @@ -25,11 +25,21 @@ use diesel_async::{ }, }; use diesel_migrations::EmbeddedMigrations; +use futures_util::{future::BoxFuture, FutureExt}; use lemmy_utils::{error::LemmyError, settings::structs::Settings}; use once_cell::sync::Lazy; use regex::Regex; -use std::{env, env::VarError, time::Duration}; -use tracing::info; +use rustls::{ + client::{ServerCertVerified, ServerCertVerifier}, + ServerName, +}; +use std::{ + env, + env::VarError, + sync::Arc, + time::{Duration, SystemTime}, +}; +use tracing::{error, info}; use url::Url; const FETCH_LIMIT_DEFAULT: i64 = 10; @@ -136,7 +146,15 @@ pub fn diesel_option_overwrite_to_url_create( async fn build_db_pool_settings_opt(settings: Option<&Settings>) -> Result { let db_url = get_database_url(settings); let pool_size = settings.map(|s| s.database.pool_size).unwrap_or(5); - let manager = AsyncDieselConnectionManager::::new(&db_url); + // We only support TLS with sslmode=require currently + let tls_enabled = db_url.contains("sslmode=require"); + let manager = if tls_enabled { + // diesel-async does not support any TLS connections out of the box, so we need to manually + // provide a setup function which handles creating the connection + AsyncDieselConnectionManager::::new_with_setup(&db_url, establish_connection) + } else { + AsyncDieselConnectionManager::::new(&db_url) + }; let pool = Pool::builder(manager) .max_size(pool_size) .wait_timeout(POOL_TIMEOUT) @@ -153,6 +171,44 @@ async fn build_db_pool_settings_opt(settings: Option<&Settings>) -> Result BoxFuture> { + let fut = async { + let rustls_config = rustls::ClientConfig::builder() + .with_safe_defaults() + .with_custom_certificate_verifier(Arc::new(NoCertVerifier {})) + .with_no_client_auth(); + + let tls = tokio_postgres_rustls::MakeRustlsConnect::new(rustls_config); + let (client, conn) = tokio_postgres::connect(config, tls) + .await + .map_err(|e| ConnectionError::BadConnection(e.to_string()))?; + tokio::spawn(async move { + if let Err(e) = conn.await { + error!("Database connection failed: {e}"); + } + }); + AsyncPgConnection::try_from(client).await + }; + fut.boxed() +} + +struct NoCertVerifier {} + +impl ServerCertVerifier for NoCertVerifier { + fn verify_server_cert( + &self, + _end_entity: &rustls::Certificate, + _intermediates: &[rustls::Certificate], + _server_name: &ServerName, + _scts: &mut dyn Iterator, + _ocsp_response: &[u8], + _now: SystemTime, + ) -> Result { + // Will verify all (even invalid) certs without any checks (sslmode=require) + Ok(ServerCertVerified::assertion()) + } +} + pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!(); pub fn run_migrations(db_url: &str) { From 95378785195c552a6618acb44d7ca31b084ee480 Mon Sep 17 00:00:00 2001 From: TKilFree Date: Fri, 23 Jun 2023 10:47:12 +0100 Subject: [PATCH 04/45] feat: re-added captcha checks (#3249) --- Cargo.lock | 2 + crates/api/Cargo.toml | 1 + crates/api/src/lib.rs | 16 ++ crates/api/src/local_user/get_captcha.rs | 53 ++++++ crates/api/src/local_user/mod.rs | 1 + crates/api_crud/Cargo.toml | 1 + crates/api_crud/src/user/create.rs | 18 ++ crates/db_schema/src/diesel_ltree.patch | 26 +-- crates/db_schema/src/impls/captcha_answer.rs | 164 ++++++++++++++++++ crates/db_schema/src/impls/mod.rs | 1 + crates/db_schema/src/schema.rs | 9 + crates/db_schema/src/source/captcha_answer.rs | 14 ++ crates/db_schema/src/source/mod.rs | 1 + .../2023-06-21-153242_add_captcha/down.sql | 1 + .../2023-06-21-153242_add_captcha/up.sql | 5 + src/api_routes_http.rs | 7 + 16 files changed, 300 insertions(+), 20 deletions(-) create mode 100644 crates/api/src/local_user/get_captcha.rs create mode 100644 crates/db_schema/src/impls/captcha_answer.rs create mode 100644 crates/db_schema/src/source/captcha_answer.rs create mode 100644 migrations/2023-06-21-153242_add_captcha/down.sql create mode 100644 migrations/2023-06-21-153242_add_captcha/up.sql diff --git a/Cargo.lock b/Cargo.lock index cee02f798..08437ecf7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2526,6 +2526,7 @@ dependencies = [ "base64 0.13.1", "bcrypt", "captcha", + "chrono", "lemmy_api_common", "lemmy_db_schema", "lemmy_db_views", @@ -2576,6 +2577,7 @@ dependencies = [ "actix-web", "async-trait", "bcrypt", + "chrono", "lemmy_api_common", "lemmy_db_schema", "lemmy_db_views", diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index 2488f2c2c..ca792809b 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -29,6 +29,7 @@ async-trait = { workspace = true } captcha = { workspace = true } anyhow = { workspace = true } tracing = { workspace = true } +chrono = { workspace = true } [dev-dependencies] serial_test = { workspace = true } diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 9ff1677d0..615a8a314 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -1,4 +1,5 @@ use actix_web::web::Data; +use captcha::Captcha; use lemmy_api_common::{context::LemmyContext, utils::local_site_to_slur_regex}; use lemmy_db_schema::source::local_site::LocalSite; use lemmy_utils::{error::LemmyError, utils::slurs::check_slurs}; @@ -20,6 +21,21 @@ pub trait Perform { async fn perform(&self, context: &Data) -> Result; } +/// Converts the captcha to a base64 encoded wav audio file +pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> String { + let letters = captcha.as_wav(); + + let mut concat_letters: Vec = Vec::new(); + + for letter in letters { + let bytes = letter.unwrap_or_default(); + concat_letters.extend(bytes); + } + + // Convert to base64 + base64::encode(concat_letters) +} + /// Check size of report and remove whitespace pub(crate) fn check_report_reason(reason: &str, local_site: &LocalSite) -> Result<(), LemmyError> { let slur_regex = &local_site_to_slur_regex(local_site); diff --git a/crates/api/src/local_user/get_captcha.rs b/crates/api/src/local_user/get_captcha.rs new file mode 100644 index 000000000..6dbc34823 --- /dev/null +++ b/crates/api/src/local_user/get_captcha.rs @@ -0,0 +1,53 @@ +use crate::{captcha_as_wav_base64, Perform}; +use actix_web::web::Data; +use captcha::{gen, Difficulty}; +use chrono::Duration; +use lemmy_api_common::{ + context::LemmyContext, + person::{CaptchaResponse, GetCaptcha, GetCaptchaResponse}, +}; +use lemmy_db_schema::{ + source::{captcha_answer::CaptchaAnswer, local_site::LocalSite}, + utils::naive_now, +}; +use lemmy_utils::error::LemmyError; + +#[async_trait::async_trait(?Send)] +impl Perform for GetCaptcha { + type Response = GetCaptchaResponse; + + #[tracing::instrument(skip(context))] + async fn perform(&self, context: &Data) -> Result { + let local_site = LocalSite::read(context.pool()).await?; + + if !local_site.captcha_enabled { + return Ok(GetCaptchaResponse { ok: None }); + } + + let captcha = gen(match local_site.captcha_difficulty.as_str() { + "easy" => Difficulty::Easy, + "hard" => Difficulty::Hard, + _ => Difficulty::Medium, + }); + + let answer = captcha.chars_as_string(); + + let png = captcha.as_base64().expect("failed to generate captcha"); + + let uuid = uuid::Uuid::new_v4().to_string(); + + let wav = captcha_as_wav_base64(&captcha); + + let captcha: CaptchaAnswer = CaptchaAnswer { + answer, + uuid: uuid.clone(), + expires: naive_now() + Duration::minutes(10), // expires in 10 minutes + }; + // Stores the captcha item in the db + CaptchaAnswer::insert(context.pool(), &captcha).await?; + + Ok(GetCaptchaResponse { + ok: Some(CaptchaResponse { png, wav, uuid }), + }) + } +} diff --git a/crates/api/src/local_user/mod.rs b/crates/api/src/local_user/mod.rs index 9244f825d..3a92beda5 100644 --- a/crates/api/src/local_user/mod.rs +++ b/crates/api/src/local_user/mod.rs @@ -3,6 +3,7 @@ mod ban_person; mod block; mod change_password; mod change_password_after_reset; +mod get_captcha; mod list_banned; mod login; mod notifications; diff --git a/crates/api_crud/Cargo.toml b/crates/api_crud/Cargo.toml index 1fb1e5a66..21320a3c3 100644 --- a/crates/api_crud/Cargo.toml +++ b/crates/api_crud/Cargo.toml @@ -22,3 +22,4 @@ tracing = { workspace = true } url = { workspace = true } async-trait = { workspace = true } webmention = "0.4.0" +chrono = { worspace = true } diff --git a/crates/api_crud/src/user/create.rs b/crates/api_crud/src/user/create.rs index f5a26f756..871a05d6f 100644 --- a/crates/api_crud/src/user/create.rs +++ b/crates/api_crud/src/user/create.rs @@ -1,6 +1,7 @@ use crate::PerformCrud; use activitypub_federation::http_signatures::generate_actor_keypair; use actix_web::web::Data; +use chrono::NaiveDateTime; use lemmy_api_common::{ context::LemmyContext, person::{LoginResponse, Register}, @@ -19,6 +20,7 @@ use lemmy_api_common::{ use lemmy_db_schema::{ aggregates::structs::PersonAggregates, source::{ + captcha_answer::CaptchaAnswer, local_user::{LocalUser, LocalUserInsertForm}, person::{Person, PersonInsertForm}, registration_application::{RegistrationApplication, RegistrationApplicationInsertForm}, @@ -71,6 +73,22 @@ impl PerformCrud for Register { return Err(LemmyError::from_message("passwords_dont_match")); } + if local_site.site_setup && local_site.captcha_enabled { + let check = CaptchaAnswer::check_captcha( + context.pool(), + CaptchaAnswer { + uuid: data.captcha_uuid.clone().unwrap_or_default(), + answer: data.captcha_answer.clone().unwrap_or_default(), + // not used when checking + expires: NaiveDateTime::MIN, + }, + ) + .await?; + if !check { + return Err(LemmyError::from_message("captcha_incorrect")); + } + } + let slur_regex = local_site_to_slur_regex(&local_site); check_slurs(&data.username, &slur_regex)?; check_slurs_opt(&data.answer, &slur_regex)?; diff --git a/crates/db_schema/src/diesel_ltree.patch b/crates/db_schema/src/diesel_ltree.patch index d7d49f03e..2607eb68b 100644 --- a/crates/db_schema/src/diesel_ltree.patch +++ b/crates/db_schema/src/diesel_ltree.patch @@ -1,28 +1,17 @@ -diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs -index 255c6422..f2ccf5e2 100644 ---- a/crates/db_schema/src/schema.rs -+++ b/crates/db_schema/src/schema.rs -@@ -2,16 +2,12 @@ - - pub mod sql_types { - #[derive(diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "listing_type_enum"))] +--- schema.rs 2023-06-21 22:25:50.252384233 +0100 ++++ "schema copy.rs" 2023-06-21 22:26:50.452378651 +0100 +@@ -6,10 +6,6 @@ pub struct ListingTypeEnum; -- #[derive(diesel::sql_types::SqlType)] + #[derive(diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "ltree"))] - pub struct Ltree; - - #[derive(diesel::sql_types::SqlType)] +- #[derive(diesel::sql_types::SqlType)] #[diesel(postgres_type(name = "registration_mode_enum"))] pub struct RegistrationModeEnum; - #[derive(diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "sort_type_enum"))] -@@ -67,13 +63,13 @@ diesel::table! { - when_ -> Timestamp, - } - } +@@ -78,7 +74,7 @@ diesel::table! { use diesel::sql_types::*; @@ -31,6 +20,3 @@ index 255c6422..f2ccf5e2 100644 comment (id) { id -> Int4, - creator_id -> Int4, - post_id -> Int4, - content -> Text, diff --git a/crates/db_schema/src/impls/captcha_answer.rs b/crates/db_schema/src/impls/captcha_answer.rs new file mode 100644 index 000000000..afd18181c --- /dev/null +++ b/crates/db_schema/src/impls/captcha_answer.rs @@ -0,0 +1,164 @@ +use crate::{ + schema::captcha_answer, + source::captcha_answer::CaptchaAnswer, + utils::{functions::lower, get_conn, naive_now, DbPool}, +}; +use diesel::{ + delete, + dsl::exists, + insert_into, + result::Error, + select, + ExpressionMethods, + QueryDsl, +}; +use diesel_async::RunQueryDsl; + +impl CaptchaAnswer { + pub async fn insert(pool: &DbPool, captcha: &CaptchaAnswer) -> Result { + let conn = &mut get_conn(pool).await?; + + insert_into(captcha_answer::table) + .values(captcha) + .get_result::(conn) + .await + } + + pub async fn check_captcha(pool: &DbPool, to_check: CaptchaAnswer) -> Result { + let conn = &mut get_conn(pool).await?; + + // delete any expired captchas + delete(captcha_answer::table.filter(captcha_answer::expires.lt(&naive_now()))) + .execute(conn) + .await?; + + // fetch requested captcha + let captcha_exists = select(exists( + captcha_answer::dsl::captcha_answer + .filter((captcha_answer::dsl::uuid).eq(to_check.uuid.clone())) + .filter(lower(captcha_answer::dsl::answer).eq(to_check.answer.to_lowercase().clone())), + )) + .get_result::(conn) + .await?; + + // delete checked captcha + delete(captcha_answer::table.filter(captcha_answer::uuid.eq(to_check.uuid.clone()))) + .execute(conn) + .await?; + + Ok(captcha_exists) + } +} + +#[cfg(test)] +mod tests { + use crate::{ + source::captcha_answer::CaptchaAnswer, + utils::{build_db_pool_for_tests, naive_now}, + }; + use chrono::Duration; + use serial_test::serial; + + #[tokio::test] + #[serial] + async fn test_captcha_happy_path() { + let pool = &build_db_pool_for_tests().await; + + let captcha_a_id = "a".to_string(); + + let _ = CaptchaAnswer::insert( + pool, + &CaptchaAnswer { + uuid: captcha_a_id.clone(), + answer: "XYZ".to_string(), + expires: naive_now() + Duration::minutes(10), + }, + ) + .await; + + let result = CaptchaAnswer::check_captcha( + pool, + CaptchaAnswer { + uuid: captcha_a_id.clone(), + answer: "xyz".to_string(), + expires: chrono::NaiveDateTime::MIN, + }, + ) + .await; + + assert!(result.is_ok()); + assert!(result.unwrap()); + } + + #[tokio::test] + #[serial] + async fn test_captcha_repeat_answer_fails() { + let pool = &build_db_pool_for_tests().await; + + let captcha_a_id = "a".to_string(); + + let _ = CaptchaAnswer::insert( + pool, + &CaptchaAnswer { + uuid: captcha_a_id.clone(), + answer: "XYZ".to_string(), + expires: naive_now() + Duration::minutes(10), + }, + ) + .await; + + let result = CaptchaAnswer::check_captcha( + pool, + CaptchaAnswer { + uuid: captcha_a_id.clone(), + answer: "xyz".to_string(), + expires: chrono::NaiveDateTime::MIN, + }, + ) + .await; + + let result_repeat = CaptchaAnswer::check_captcha( + pool, + CaptchaAnswer { + uuid: captcha_a_id.clone(), + answer: "xyz".to_string(), + expires: chrono::NaiveDateTime::MIN, + }, + ) + .await; + + assert!(result_repeat.is_ok()); + assert!(!result_repeat.unwrap()); + } + + #[tokio::test] + #[serial] + async fn test_captcha_expired_fails() { + let pool = &build_db_pool_for_tests().await; + + let expired_id = "already_expired".to_string(); + + let _ = CaptchaAnswer::insert( + pool, + &CaptchaAnswer { + uuid: expired_id.clone(), + answer: "xyz".to_string(), + expires: naive_now() - Duration::seconds(1), + }, + ) + .await; + + let expired_result = CaptchaAnswer::check_captcha( + pool, + CaptchaAnswer { + uuid: expired_id.clone(), + answer: "xyz".to_string(), + expires: chrono::NaiveDateTime::MIN, + }, + ) + .await; + + assert!(expired_result.is_ok()); + assert!(!expired_result.unwrap()); + } +} diff --git a/crates/db_schema/src/impls/mod.rs b/crates/db_schema/src/impls/mod.rs index 915d1c8e2..f13004d01 100644 --- a/crates/db_schema/src/impls/mod.rs +++ b/crates/db_schema/src/impls/mod.rs @@ -1,5 +1,6 @@ pub mod activity; pub mod actor_language; +pub mod captcha_answer; pub mod comment; pub mod comment_reply; pub mod comment_report; diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index ac4ddc47a..f244ae664 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -64,6 +64,14 @@ diesel::table! { } } +diesel::table! { + captcha_answer (uuid) { + uuid -> Text, + answer -> Text, + expires -> Timestamp, + } +} + diesel::table! { use diesel::sql_types::{Bool, Int4, Nullable, Text, Timestamp, Varchar}; use diesel_ltree::sql_types::Ltree; @@ -916,6 +924,7 @@ diesel::allow_tables_to_appear_in_same_query!( admin_purge_community, admin_purge_person, admin_purge_post, + captcha_answer, comment, comment_aggregates, comment_like, diff --git a/crates/db_schema/src/source/captcha_answer.rs b/crates/db_schema/src/source/captcha_answer.rs new file mode 100644 index 000000000..113b7c96a --- /dev/null +++ b/crates/db_schema/src/source/captcha_answer.rs @@ -0,0 +1,14 @@ +#[cfg(feature = "full")] +use crate::schema::captcha_answer; +use serde::{Deserialize, Serialize}; +use serde_with::skip_serializing_none; + +#[skip_serializing_none] +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[cfg_attr(feature = "full", derive(Queryable, Insertable, AsChangeset))] +#[cfg_attr(feature = "full", diesel(table_name = captcha_answer))] +pub struct CaptchaAnswer { + pub uuid: String, + pub answer: String, + pub expires: chrono::NaiveDateTime, +} diff --git a/crates/db_schema/src/source/mod.rs b/crates/db_schema/src/source/mod.rs index 9aab4b90b..926e23e73 100644 --- a/crates/db_schema/src/source/mod.rs +++ b/crates/db_schema/src/source/mod.rs @@ -1,6 +1,7 @@ #[cfg(feature = "full")] pub mod activity; pub mod actor_language; +pub mod captcha_answer; pub mod comment; pub mod comment_reply; pub mod comment_report; diff --git a/migrations/2023-06-21-153242_add_captcha/down.sql b/migrations/2023-06-21-153242_add_captcha/down.sql new file mode 100644 index 000000000..4e5b83042 --- /dev/null +++ b/migrations/2023-06-21-153242_add_captcha/down.sql @@ -0,0 +1 @@ +drop table captcha_answer; \ No newline at end of file diff --git a/migrations/2023-06-21-153242_add_captcha/up.sql b/migrations/2023-06-21-153242_add_captcha/up.sql new file mode 100644 index 000000000..71467be61 --- /dev/null +++ b/migrations/2023-06-21-153242_add_captcha/up.sql @@ -0,0 +1,5 @@ +create table captcha_answer ( + uuid text not null primary key, + answer text not null, + expires timestamp not null +); diff --git a/src/api_routes_http.rs b/src/api_routes_http.rs index a2abfa690..375630a92 100644 --- a/src/api_routes_http.rs +++ b/src/api_routes_http.rs @@ -38,6 +38,7 @@ use lemmy_api_common::{ ChangePassword, DeleteAccount, GetBannedPersons, + GetCaptcha, GetPersonDetails, GetPersonMentions, GetReplies, @@ -272,6 +273,12 @@ pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) { .wrap(rate_limit.register()) .route(web::post().to(route_post_crud::)), ) + .service( + // Handle captcha separately + web::resource("/user/get_captcha") + .wrap(rate_limit.post()) + .route(web::get().to(route_get::)), + ) // User actions .service( web::scope("/user") From aea5f6a38b9ae3e913e261dc4b978c2857292703 Mon Sep 17 00:00:00 2001 From: Kresten Laust Date: Fri, 23 Jun 2023 11:50:35 +0200 Subject: [PATCH 05/45] Reversed requirement question logic (#3283) * Reversed requirement question logic * Changed required state to 'true' --- .github/ISSUE_TEMPLATE/BUG_REPORT.yml | 4 ++-- .github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/BUG_REPORT.yml b/.github/ISSUE_TEMPLATE/BUG_REPORT.yml index 206b4da85..a4028afd0 100644 --- a/.github/ISSUE_TEMPLATE/BUG_REPORT.yml +++ b/.github/ISSUE_TEMPLATE/BUG_REPORT.yml @@ -20,8 +20,8 @@ body: required: true - label: Is this only a single bug? Do not put multiple bugs in one issue. required: true - - label: Is this a UI / front end issue? Use the [lemmy-ui](https://github.com/LemmyNet/lemmy-ui) repo. - required: false + - label: Is this a backend issue? Use the [lemmy-ui](https://github.com/LemmyNet/lemmy-ui) repo for UI / frontend issues. + required: true - type: textarea id: summary attributes: diff --git a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml index 5f35bb030..40ef2caf3 100644 --- a/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml +++ b/.github/ISSUE_TEMPLATE/FEATURE_REQUEST.yml @@ -18,8 +18,8 @@ body: required: true - label: Is this only a feature request? Do not put multiple feature requests in one issue. required: true - - label: Is this a UI / front end issue? Use the [lemmy-ui](https://github.com/LemmyNet/lemmy-ui) repo. - required: false + - label: Is this a backend issue? Use the [lemmy-ui](https://github.com/LemmyNet/lemmy-ui) repo for UI / frontend issues. + required: true - type: textarea id: problem attributes: From b6cd1bde8eae74d22c0c3c80ee7990b413d0ab27 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 23 Jun 2023 06:53:46 -0400 Subject: [PATCH 06/45] Fixing removed posts showing. Fixes #2875 (#3279) * Fixing removed posts showing. Fixes #2875 * Fixing clippy. --- crates/db_views/src/post_view.rs | 58 +++++++++++++++++++-- crates/db_views_actor/src/community_view.rs | 6 +-- 2 files changed, 55 insertions(+), 9 deletions(-) diff --git a/crates/db_views/src/post_view.rs b/crates/db_views/src/post_view.rs index 79cc875cc..14cf7fe19 100644 --- a/crates/db_views/src/post_view.rs +++ b/crates/db_views/src/post_view.rs @@ -147,10 +147,12 @@ impl PostView { .into_boxed(); // Hide deleted and removed for non-admins or mods - if !is_mod_or_admin.unwrap_or(true) { + if !is_mod_or_admin.unwrap_or(false) { query = query .filter(community::removed.eq(false)) - .filter(community::deleted.eq(false)); + .filter(community::deleted.eq(false)) + .filter(post::removed.eq(false)) + .filter(post::deleted.eq(false)); } let ( @@ -305,10 +307,12 @@ impl<'a> PostQuery<'a> { // Hide deleted and removed for non-admins or mods // TODO This eventually needs to show posts where you are the creator - if !self.is_mod_or_admin.unwrap_or(true) { + if !self.is_mod_or_admin.unwrap_or(false) { query = query .filter(community::removed.eq(false)) - .filter(community::deleted.eq(false)); + .filter(community::deleted.eq(false)) + .filter(post::removed.eq(false)) + .filter(post::deleted.eq(false)); } if self.community_id.is_none() { @@ -475,7 +479,7 @@ mod tests { local_user::{LocalUser, LocalUserInsertForm, LocalUserUpdateForm}, person::{Person, PersonInsertForm}, person_block::{PersonBlock, PersonBlockForm}, - post::{Post, PostInsertForm, PostLike, PostLikeForm}, + post::{Post, PostInsertForm, PostLike, PostLikeForm, PostUpdateForm}, }, traits::{Blockable, Crud, Likeable}, utils::{build_db_pool_for_tests, DbPool}, @@ -870,6 +874,50 @@ mod tests { cleanup(data, pool).await; } + #[tokio::test] + #[serial] + async fn post_listings_deleted() { + let pool = &build_db_pool_for_tests().await; + let data = init_data(pool).await; + + // Delete the post + Post::update( + pool, + data.inserted_post.id, + &PostUpdateForm::builder().deleted(Some(true)).build(), + ) + .await + .unwrap(); + + // Make sure you don't see the deleted post in the results + let post_listings_no_admin = PostQuery::builder() + .pool(pool) + .sort(Some(SortType::New)) + .local_user(Some(&data.inserted_local_user)) + .is_mod_or_admin(Some(false)) + .build() + .list() + .await + .unwrap(); + + assert_eq!(1, post_listings_no_admin.len()); + + // Make sure they see both + let post_listings_is_admin = PostQuery::builder() + .pool(pool) + .sort(Some(SortType::New)) + .local_user(Some(&data.inserted_local_user)) + .is_mod_or_admin(Some(true)) + .build() + .list() + .await + .unwrap(); + + assert_eq!(2, post_listings_is_admin.len()); + + cleanup(data, pool).await; + } + async fn cleanup(data: Data, pool: &DbPool) { let num_deleted = Post::delete(pool, data.inserted_post.id).await.unwrap(); Community::delete(pool, data.inserted_community.id) diff --git a/crates/db_views_actor/src/community_view.rs b/crates/db_views_actor/src/community_view.rs index ee6066c22..c4f5920b7 100644 --- a/crates/db_views_actor/src/community_view.rs +++ b/crates/db_views_actor/src/community_view.rs @@ -69,7 +69,7 @@ impl CommunityView { .into_boxed(); // Hide deleted and removed for non-admins or mods - if !is_mod_or_admin.unwrap_or(true) { + if !is_mod_or_admin.unwrap_or(false) { query = query .filter(community::removed.eq(false)) .filter(community::deleted.eq(false)); @@ -170,7 +170,7 @@ impl<'a> CommunityQuery<'a> { }; // Hide deleted and removed for non-admins or mods - if !self.is_mod_or_admin.unwrap_or(true) { + if !self.is_mod_or_admin.unwrap_or(false) { query = query .filter(community::removed.eq(false)) .filter(community::deleted.eq(false)) @@ -213,8 +213,6 @@ impl<'a> CommunityQuery<'a> { let res = query .limit(limit) .offset(offset) - .filter(community::removed.eq(false)) - .filter(community::deleted.eq(false)) .load::(conn) .await?; From 37a47de3a871830ad002ec1f5c4da672ca430518 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 23 Jun 2023 07:02:05 -0400 Subject: [PATCH 07/45] Revert "feat: re-added captcha checks (#3249)" (#3288) This reverts commit 8a086c82405bc8e2c8cb2fbbcceb10418f231d1b. --- Cargo.lock | 2 - crates/api/Cargo.toml | 1 - crates/api/src/lib.rs | 16 -- crates/api/src/local_user/get_captcha.rs | 53 ------ crates/api/src/local_user/mod.rs | 1 - crates/api_crud/Cargo.toml | 1 - crates/api_crud/src/user/create.rs | 18 -- crates/db_schema/src/diesel_ltree.patch | 26 ++- crates/db_schema/src/impls/captcha_answer.rs | 164 ------------------ crates/db_schema/src/impls/mod.rs | 1 - crates/db_schema/src/schema.rs | 9 - crates/db_schema/src/source/captcha_answer.rs | 14 -- crates/db_schema/src/source/mod.rs | 1 - .../2023-06-21-153242_add_captcha/down.sql | 1 - .../2023-06-21-153242_add_captcha/up.sql | 5 - src/api_routes_http.rs | 7 - 16 files changed, 20 insertions(+), 300 deletions(-) delete mode 100644 crates/api/src/local_user/get_captcha.rs delete mode 100644 crates/db_schema/src/impls/captcha_answer.rs delete mode 100644 crates/db_schema/src/source/captcha_answer.rs delete mode 100644 migrations/2023-06-21-153242_add_captcha/down.sql delete mode 100644 migrations/2023-06-21-153242_add_captcha/up.sql diff --git a/Cargo.lock b/Cargo.lock index 08437ecf7..cee02f798 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2526,7 +2526,6 @@ dependencies = [ "base64 0.13.1", "bcrypt", "captcha", - "chrono", "lemmy_api_common", "lemmy_db_schema", "lemmy_db_views", @@ -2577,7 +2576,6 @@ dependencies = [ "actix-web", "async-trait", "bcrypt", - "chrono", "lemmy_api_common", "lemmy_db_schema", "lemmy_db_views", diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index ca792809b..2488f2c2c 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -29,7 +29,6 @@ async-trait = { workspace = true } captcha = { workspace = true } anyhow = { workspace = true } tracing = { workspace = true } -chrono = { workspace = true } [dev-dependencies] serial_test = { workspace = true } diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 615a8a314..9ff1677d0 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -1,5 +1,4 @@ use actix_web::web::Data; -use captcha::Captcha; use lemmy_api_common::{context::LemmyContext, utils::local_site_to_slur_regex}; use lemmy_db_schema::source::local_site::LocalSite; use lemmy_utils::{error::LemmyError, utils::slurs::check_slurs}; @@ -21,21 +20,6 @@ pub trait Perform { async fn perform(&self, context: &Data) -> Result; } -/// Converts the captcha to a base64 encoded wav audio file -pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> String { - let letters = captcha.as_wav(); - - let mut concat_letters: Vec = Vec::new(); - - for letter in letters { - let bytes = letter.unwrap_or_default(); - concat_letters.extend(bytes); - } - - // Convert to base64 - base64::encode(concat_letters) -} - /// Check size of report and remove whitespace pub(crate) fn check_report_reason(reason: &str, local_site: &LocalSite) -> Result<(), LemmyError> { let slur_regex = &local_site_to_slur_regex(local_site); diff --git a/crates/api/src/local_user/get_captcha.rs b/crates/api/src/local_user/get_captcha.rs deleted file mode 100644 index 6dbc34823..000000000 --- a/crates/api/src/local_user/get_captcha.rs +++ /dev/null @@ -1,53 +0,0 @@ -use crate::{captcha_as_wav_base64, Perform}; -use actix_web::web::Data; -use captcha::{gen, Difficulty}; -use chrono::Duration; -use lemmy_api_common::{ - context::LemmyContext, - person::{CaptchaResponse, GetCaptcha, GetCaptchaResponse}, -}; -use lemmy_db_schema::{ - source::{captcha_answer::CaptchaAnswer, local_site::LocalSite}, - utils::naive_now, -}; -use lemmy_utils::error::LemmyError; - -#[async_trait::async_trait(?Send)] -impl Perform for GetCaptcha { - type Response = GetCaptchaResponse; - - #[tracing::instrument(skip(context))] - async fn perform(&self, context: &Data) -> Result { - let local_site = LocalSite::read(context.pool()).await?; - - if !local_site.captcha_enabled { - return Ok(GetCaptchaResponse { ok: None }); - } - - let captcha = gen(match local_site.captcha_difficulty.as_str() { - "easy" => Difficulty::Easy, - "hard" => Difficulty::Hard, - _ => Difficulty::Medium, - }); - - let answer = captcha.chars_as_string(); - - let png = captcha.as_base64().expect("failed to generate captcha"); - - let uuid = uuid::Uuid::new_v4().to_string(); - - let wav = captcha_as_wav_base64(&captcha); - - let captcha: CaptchaAnswer = CaptchaAnswer { - answer, - uuid: uuid.clone(), - expires: naive_now() + Duration::minutes(10), // expires in 10 minutes - }; - // Stores the captcha item in the db - CaptchaAnswer::insert(context.pool(), &captcha).await?; - - Ok(GetCaptchaResponse { - ok: Some(CaptchaResponse { png, wav, uuid }), - }) - } -} diff --git a/crates/api/src/local_user/mod.rs b/crates/api/src/local_user/mod.rs index 3a92beda5..9244f825d 100644 --- a/crates/api/src/local_user/mod.rs +++ b/crates/api/src/local_user/mod.rs @@ -3,7 +3,6 @@ mod ban_person; mod block; mod change_password; mod change_password_after_reset; -mod get_captcha; mod list_banned; mod login; mod notifications; diff --git a/crates/api_crud/Cargo.toml b/crates/api_crud/Cargo.toml index 21320a3c3..1fb1e5a66 100644 --- a/crates/api_crud/Cargo.toml +++ b/crates/api_crud/Cargo.toml @@ -22,4 +22,3 @@ tracing = { workspace = true } url = { workspace = true } async-trait = { workspace = true } webmention = "0.4.0" -chrono = { worspace = true } diff --git a/crates/api_crud/src/user/create.rs b/crates/api_crud/src/user/create.rs index 871a05d6f..f5a26f756 100644 --- a/crates/api_crud/src/user/create.rs +++ b/crates/api_crud/src/user/create.rs @@ -1,7 +1,6 @@ use crate::PerformCrud; use activitypub_federation::http_signatures::generate_actor_keypair; use actix_web::web::Data; -use chrono::NaiveDateTime; use lemmy_api_common::{ context::LemmyContext, person::{LoginResponse, Register}, @@ -20,7 +19,6 @@ use lemmy_api_common::{ use lemmy_db_schema::{ aggregates::structs::PersonAggregates, source::{ - captcha_answer::CaptchaAnswer, local_user::{LocalUser, LocalUserInsertForm}, person::{Person, PersonInsertForm}, registration_application::{RegistrationApplication, RegistrationApplicationInsertForm}, @@ -73,22 +71,6 @@ impl PerformCrud for Register { return Err(LemmyError::from_message("passwords_dont_match")); } - if local_site.site_setup && local_site.captcha_enabled { - let check = CaptchaAnswer::check_captcha( - context.pool(), - CaptchaAnswer { - uuid: data.captcha_uuid.clone().unwrap_or_default(), - answer: data.captcha_answer.clone().unwrap_or_default(), - // not used when checking - expires: NaiveDateTime::MIN, - }, - ) - .await?; - if !check { - return Err(LemmyError::from_message("captcha_incorrect")); - } - } - let slur_regex = local_site_to_slur_regex(&local_site); check_slurs(&data.username, &slur_regex)?; check_slurs_opt(&data.answer, &slur_regex)?; diff --git a/crates/db_schema/src/diesel_ltree.patch b/crates/db_schema/src/diesel_ltree.patch index 2607eb68b..d7d49f03e 100644 --- a/crates/db_schema/src/diesel_ltree.patch +++ b/crates/db_schema/src/diesel_ltree.patch @@ -1,17 +1,28 @@ ---- schema.rs 2023-06-21 22:25:50.252384233 +0100 -+++ "schema copy.rs" 2023-06-21 22:26:50.452378651 +0100 -@@ -6,10 +6,6 @@ +diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs +index 255c6422..f2ccf5e2 100644 +--- a/crates/db_schema/src/schema.rs ++++ b/crates/db_schema/src/schema.rs +@@ -2,16 +2,12 @@ + + pub mod sql_types { + #[derive(diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "listing_type_enum"))] pub struct ListingTypeEnum; - #[derive(diesel::sql_types::SqlType)] +- #[derive(diesel::sql_types::SqlType)] - #[diesel(postgres_type(name = "ltree"))] - pub struct Ltree; - -- #[derive(diesel::sql_types::SqlType)] + #[derive(diesel::sql_types::SqlType)] #[diesel(postgres_type(name = "registration_mode_enum"))] pub struct RegistrationModeEnum; -@@ -78,7 +74,7 @@ + #[derive(diesel::sql_types::SqlType)] + #[diesel(postgres_type(name = "sort_type_enum"))] +@@ -67,13 +63,13 @@ diesel::table! { + when_ -> Timestamp, + } + } diesel::table! { use diesel::sql_types::*; @@ -20,3 +31,6 @@ comment (id) { id -> Int4, + creator_id -> Int4, + post_id -> Int4, + content -> Text, diff --git a/crates/db_schema/src/impls/captcha_answer.rs b/crates/db_schema/src/impls/captcha_answer.rs deleted file mode 100644 index afd18181c..000000000 --- a/crates/db_schema/src/impls/captcha_answer.rs +++ /dev/null @@ -1,164 +0,0 @@ -use crate::{ - schema::captcha_answer, - source::captcha_answer::CaptchaAnswer, - utils::{functions::lower, get_conn, naive_now, DbPool}, -}; -use diesel::{ - delete, - dsl::exists, - insert_into, - result::Error, - select, - ExpressionMethods, - QueryDsl, -}; -use diesel_async::RunQueryDsl; - -impl CaptchaAnswer { - pub async fn insert(pool: &DbPool, captcha: &CaptchaAnswer) -> Result { - let conn = &mut get_conn(pool).await?; - - insert_into(captcha_answer::table) - .values(captcha) - .get_result::(conn) - .await - } - - pub async fn check_captcha(pool: &DbPool, to_check: CaptchaAnswer) -> Result { - let conn = &mut get_conn(pool).await?; - - // delete any expired captchas - delete(captcha_answer::table.filter(captcha_answer::expires.lt(&naive_now()))) - .execute(conn) - .await?; - - // fetch requested captcha - let captcha_exists = select(exists( - captcha_answer::dsl::captcha_answer - .filter((captcha_answer::dsl::uuid).eq(to_check.uuid.clone())) - .filter(lower(captcha_answer::dsl::answer).eq(to_check.answer.to_lowercase().clone())), - )) - .get_result::(conn) - .await?; - - // delete checked captcha - delete(captcha_answer::table.filter(captcha_answer::uuid.eq(to_check.uuid.clone()))) - .execute(conn) - .await?; - - Ok(captcha_exists) - } -} - -#[cfg(test)] -mod tests { - use crate::{ - source::captcha_answer::CaptchaAnswer, - utils::{build_db_pool_for_tests, naive_now}, - }; - use chrono::Duration; - use serial_test::serial; - - #[tokio::test] - #[serial] - async fn test_captcha_happy_path() { - let pool = &build_db_pool_for_tests().await; - - let captcha_a_id = "a".to_string(); - - let _ = CaptchaAnswer::insert( - pool, - &CaptchaAnswer { - uuid: captcha_a_id.clone(), - answer: "XYZ".to_string(), - expires: naive_now() + Duration::minutes(10), - }, - ) - .await; - - let result = CaptchaAnswer::check_captcha( - pool, - CaptchaAnswer { - uuid: captcha_a_id.clone(), - answer: "xyz".to_string(), - expires: chrono::NaiveDateTime::MIN, - }, - ) - .await; - - assert!(result.is_ok()); - assert!(result.unwrap()); - } - - #[tokio::test] - #[serial] - async fn test_captcha_repeat_answer_fails() { - let pool = &build_db_pool_for_tests().await; - - let captcha_a_id = "a".to_string(); - - let _ = CaptchaAnswer::insert( - pool, - &CaptchaAnswer { - uuid: captcha_a_id.clone(), - answer: "XYZ".to_string(), - expires: naive_now() + Duration::minutes(10), - }, - ) - .await; - - let result = CaptchaAnswer::check_captcha( - pool, - CaptchaAnswer { - uuid: captcha_a_id.clone(), - answer: "xyz".to_string(), - expires: chrono::NaiveDateTime::MIN, - }, - ) - .await; - - let result_repeat = CaptchaAnswer::check_captcha( - pool, - CaptchaAnswer { - uuid: captcha_a_id.clone(), - answer: "xyz".to_string(), - expires: chrono::NaiveDateTime::MIN, - }, - ) - .await; - - assert!(result_repeat.is_ok()); - assert!(!result_repeat.unwrap()); - } - - #[tokio::test] - #[serial] - async fn test_captcha_expired_fails() { - let pool = &build_db_pool_for_tests().await; - - let expired_id = "already_expired".to_string(); - - let _ = CaptchaAnswer::insert( - pool, - &CaptchaAnswer { - uuid: expired_id.clone(), - answer: "xyz".to_string(), - expires: naive_now() - Duration::seconds(1), - }, - ) - .await; - - let expired_result = CaptchaAnswer::check_captcha( - pool, - CaptchaAnswer { - uuid: expired_id.clone(), - answer: "xyz".to_string(), - expires: chrono::NaiveDateTime::MIN, - }, - ) - .await; - - assert!(expired_result.is_ok()); - assert!(!expired_result.unwrap()); - } -} diff --git a/crates/db_schema/src/impls/mod.rs b/crates/db_schema/src/impls/mod.rs index f13004d01..915d1c8e2 100644 --- a/crates/db_schema/src/impls/mod.rs +++ b/crates/db_schema/src/impls/mod.rs @@ -1,6 +1,5 @@ pub mod activity; pub mod actor_language; -pub mod captcha_answer; pub mod comment; pub mod comment_reply; pub mod comment_report; diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index f244ae664..ac4ddc47a 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -64,14 +64,6 @@ diesel::table! { } } -diesel::table! { - captcha_answer (uuid) { - uuid -> Text, - answer -> Text, - expires -> Timestamp, - } -} - diesel::table! { use diesel::sql_types::{Bool, Int4, Nullable, Text, Timestamp, Varchar}; use diesel_ltree::sql_types::Ltree; @@ -924,7 +916,6 @@ diesel::allow_tables_to_appear_in_same_query!( admin_purge_community, admin_purge_person, admin_purge_post, - captcha_answer, comment, comment_aggregates, comment_like, diff --git a/crates/db_schema/src/source/captcha_answer.rs b/crates/db_schema/src/source/captcha_answer.rs deleted file mode 100644 index 113b7c96a..000000000 --- a/crates/db_schema/src/source/captcha_answer.rs +++ /dev/null @@ -1,14 +0,0 @@ -#[cfg(feature = "full")] -use crate::schema::captcha_answer; -use serde::{Deserialize, Serialize}; -use serde_with::skip_serializing_none; - -#[skip_serializing_none] -#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] -#[cfg_attr(feature = "full", derive(Queryable, Insertable, AsChangeset))] -#[cfg_attr(feature = "full", diesel(table_name = captcha_answer))] -pub struct CaptchaAnswer { - pub uuid: String, - pub answer: String, - pub expires: chrono::NaiveDateTime, -} diff --git a/crates/db_schema/src/source/mod.rs b/crates/db_schema/src/source/mod.rs index 926e23e73..9aab4b90b 100644 --- a/crates/db_schema/src/source/mod.rs +++ b/crates/db_schema/src/source/mod.rs @@ -1,7 +1,6 @@ #[cfg(feature = "full")] pub mod activity; pub mod actor_language; -pub mod captcha_answer; pub mod comment; pub mod comment_reply; pub mod comment_report; diff --git a/migrations/2023-06-21-153242_add_captcha/down.sql b/migrations/2023-06-21-153242_add_captcha/down.sql deleted file mode 100644 index 4e5b83042..000000000 --- a/migrations/2023-06-21-153242_add_captcha/down.sql +++ /dev/null @@ -1 +0,0 @@ -drop table captcha_answer; \ No newline at end of file diff --git a/migrations/2023-06-21-153242_add_captcha/up.sql b/migrations/2023-06-21-153242_add_captcha/up.sql deleted file mode 100644 index 71467be61..000000000 --- a/migrations/2023-06-21-153242_add_captcha/up.sql +++ /dev/null @@ -1,5 +0,0 @@ -create table captcha_answer ( - uuid text not null primary key, - answer text not null, - expires timestamp not null -); diff --git a/src/api_routes_http.rs b/src/api_routes_http.rs index 375630a92..a2abfa690 100644 --- a/src/api_routes_http.rs +++ b/src/api_routes_http.rs @@ -38,7 +38,6 @@ use lemmy_api_common::{ ChangePassword, DeleteAccount, GetBannedPersons, - GetCaptcha, GetPersonDetails, GetPersonMentions, GetReplies, @@ -273,12 +272,6 @@ pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) { .wrap(rate_limit.register()) .route(web::post().to(route_post_crud::)), ) - .service( - // Handle captcha separately - web::resource("/user/get_captcha") - .wrap(rate_limit.post()) - .route(web::get().to(route_get::)), - ) // User actions .service( web::scope("/user") From 3f6e9a7f236a464e72b43a5316f3c877525f8a03 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 23 Jun 2023 07:07:51 -0400 Subject: [PATCH 08/45] Version 0.18.0-rc.8 --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 22 +++++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cee02f798..e29f89982 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2518,7 +2518,7 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lemmy_api" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "actix-web", "anyhow", @@ -2541,7 +2541,7 @@ dependencies = [ [[package]] name = "lemmy_api_common" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "actix-rt", "actix-web", @@ -2570,7 +2570,7 @@ dependencies = [ [[package]] name = "lemmy_api_crud" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "activitypub_federation", "actix-web", @@ -2589,7 +2589,7 @@ dependencies = [ [[package]] name = "lemmy_apub" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "activitypub_federation", "actix-rt", @@ -2627,7 +2627,7 @@ dependencies = [ [[package]] name = "lemmy_db_schema" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "activitypub_federation", "async-trait", @@ -2659,7 +2659,7 @@ dependencies = [ [[package]] name = "lemmy_db_views" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "diesel", "diesel-async", @@ -2676,7 +2676,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_actor" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "diesel", "diesel-async", @@ -2689,7 +2689,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_moderator" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "diesel", "diesel-async", @@ -2701,7 +2701,7 @@ dependencies = [ [[package]] name = "lemmy_routes" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "activitypub_federation", "actix-web", @@ -2726,7 +2726,7 @@ dependencies = [ [[package]] name = "lemmy_server" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "activitypub_federation", "actix-cors", @@ -2763,7 +2763,7 @@ dependencies = [ [[package]] name = "lemmy_utils" -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" dependencies = [ "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index c05c1a57e..a86252576 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "0.18.0-rc.6" +version = "0.18.0-rc.8" edition = "2021" description = "A link aggregator for the fediverse" license = "AGPL-3.0" @@ -49,16 +49,16 @@ members = [ ] [workspace.dependencies] -lemmy_api = { version = "=0.18.0-rc.6", path = "./crates/api" } -lemmy_api_crud = { version = "=0.18.0-rc.6", path = "./crates/api_crud" } -lemmy_apub = { version = "=0.18.0-rc.6", path = "./crates/apub" } -lemmy_utils = { version = "=0.18.0-rc.6", path = "./crates/utils" } -lemmy_db_schema = { version = "=0.18.0-rc.6", path = "./crates/db_schema" } -lemmy_api_common = { version = "=0.18.0-rc.6", path = "./crates/api_common" } -lemmy_routes = { version = "=0.18.0-rc.6", path = "./crates/routes" } -lemmy_db_views = { version = "=0.18.0-rc.6", path = "./crates/db_views" } -lemmy_db_views_actor = { version = "=0.18.0-rc.6", path = "./crates/db_views_actor" } -lemmy_db_views_moderator = { version = "=0.18.0-rc.6", path = "./crates/db_views_moderator" } +lemmy_api = { version = "=0.18.0-rc.8", path = "./crates/api" } +lemmy_api_crud = { version = "=0.18.0-rc.8", path = "./crates/api_crud" } +lemmy_apub = { version = "=0.18.0-rc.8", path = "./crates/apub" } +lemmy_utils = { version = "=0.18.0-rc.8", path = "./crates/utils" } +lemmy_db_schema = { version = "=0.18.0-rc.8", path = "./crates/db_schema" } +lemmy_api_common = { version = "=0.18.0-rc.8", path = "./crates/api_common" } +lemmy_routes = { version = "=0.18.0-rc.8", path = "./crates/routes" } +lemmy_db_views = { version = "=0.18.0-rc.8", path = "./crates/db_views" } +lemmy_db_views_actor = { version = "=0.18.0-rc.8", path = "./crates/db_views_actor" } +lemmy_db_views_moderator = { version = "=0.18.0-rc.8", path = "./crates/db_views_moderator" } activitypub_federation = { version = "0.4.4", default-features = false, features = ["actix-web"] } diesel = "2.1.0" diesel_migrations = "2.1.0" From e4d78b09747ecd3700f423bce9e6b8f5db347b24 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Fri, 23 Jun 2023 08:41:20 -0400 Subject: [PATCH 09/45] Version 0.18.0 --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 22 +++++++++++----------- docker/docker-compose.yml | 4 ++-- docker/federation/docker-compose.yml | 2 +- 4 files changed, 25 insertions(+), 25 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e29f89982..9590e8f13 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2518,7 +2518,7 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lemmy_api" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "actix-web", "anyhow", @@ -2541,7 +2541,7 @@ dependencies = [ [[package]] name = "lemmy_api_common" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "actix-rt", "actix-web", @@ -2570,7 +2570,7 @@ dependencies = [ [[package]] name = "lemmy_api_crud" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "activitypub_federation", "actix-web", @@ -2589,7 +2589,7 @@ dependencies = [ [[package]] name = "lemmy_apub" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "activitypub_federation", "actix-rt", @@ -2627,7 +2627,7 @@ dependencies = [ [[package]] name = "lemmy_db_schema" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "activitypub_federation", "async-trait", @@ -2659,7 +2659,7 @@ dependencies = [ [[package]] name = "lemmy_db_views" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "diesel", "diesel-async", @@ -2676,7 +2676,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_actor" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "diesel", "diesel-async", @@ -2689,7 +2689,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_moderator" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "diesel", "diesel-async", @@ -2701,7 +2701,7 @@ dependencies = [ [[package]] name = "lemmy_routes" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "activitypub_federation", "actix-web", @@ -2726,7 +2726,7 @@ dependencies = [ [[package]] name = "lemmy_server" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "activitypub_federation", "actix-cors", @@ -2763,7 +2763,7 @@ dependencies = [ [[package]] name = "lemmy_utils" -version = "0.18.0-rc.8" +version = "0.18.0" dependencies = [ "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index a86252576..430deb082 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "0.18.0-rc.8" +version = "0.18.0" edition = "2021" description = "A link aggregator for the fediverse" license = "AGPL-3.0" @@ -49,16 +49,16 @@ members = [ ] [workspace.dependencies] -lemmy_api = { version = "=0.18.0-rc.8", path = "./crates/api" } -lemmy_api_crud = { version = "=0.18.0-rc.8", path = "./crates/api_crud" } -lemmy_apub = { version = "=0.18.0-rc.8", path = "./crates/apub" } -lemmy_utils = { version = "=0.18.0-rc.8", path = "./crates/utils" } -lemmy_db_schema = { version = "=0.18.0-rc.8", path = "./crates/db_schema" } -lemmy_api_common = { version = "=0.18.0-rc.8", path = "./crates/api_common" } -lemmy_routes = { version = "=0.18.0-rc.8", path = "./crates/routes" } -lemmy_db_views = { version = "=0.18.0-rc.8", path = "./crates/db_views" } -lemmy_db_views_actor = { version = "=0.18.0-rc.8", path = "./crates/db_views_actor" } -lemmy_db_views_moderator = { version = "=0.18.0-rc.8", path = "./crates/db_views_moderator" } +lemmy_api = { version = "=0.18.0", path = "./crates/api" } +lemmy_api_crud = { version = "=0.18.0", path = "./crates/api_crud" } +lemmy_apub = { version = "=0.18.0", path = "./crates/apub" } +lemmy_utils = { version = "=0.18.0", path = "./crates/utils" } +lemmy_db_schema = { version = "=0.18.0", path = "./crates/db_schema" } +lemmy_api_common = { version = "=0.18.0", path = "./crates/api_common" } +lemmy_routes = { version = "=0.18.0", path = "./crates/routes" } +lemmy_db_views = { version = "=0.18.0", path = "./crates/db_views" } +lemmy_db_views_actor = { version = "=0.18.0", path = "./crates/db_views_actor" } +lemmy_db_views_moderator = { version = "=0.18.0", path = "./crates/db_views_moderator" } activitypub_federation = { version = "0.4.4", default-features = false, features = ["actix-web"] } diesel = "2.1.0" diesel_migrations = "2.1.0" diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2846f547a..2033ee8c7 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -35,7 +35,7 @@ services: logging: *default-logging lemmy: - # image: dessalines/lemmy:dev + # image: dessalines/lemmy:0.18.0 # use this to build your local lemmy server image for development # run docker compose up --build build: @@ -60,7 +60,7 @@ services: logging: *default-logging lemmy-ui: - image: dessalines/lemmy-ui:0.17.1 + image: dessalines/lemmy-ui:0.18.0 # use this to build your local lemmy ui image for development # run docker compose up --build # assuming lemmy-ui is cloned besides lemmy directory diff --git a/docker/federation/docker-compose.yml b/docker/federation/docker-compose.yml index 569f264c4..b3464bacd 100644 --- a/docker/federation/docker-compose.yml +++ b/docker/federation/docker-compose.yml @@ -2,7 +2,7 @@ version: "3.7" x-ui-default: &ui-default init: true - image: dessalines/lemmy-ui:0.17.3 + image: dessalines/lemmy-ui:0.18.0 # assuming lemmy-ui is cloned besides lemmy directory # build: # context: ../../../lemmy-ui From d6b580a530563d4a2be76d077e015f9aecc75479 Mon Sep 17 00:00:00 2001 From: cetra3 Date: Mon, 26 Jun 2023 17:54:11 +0930 Subject: [PATCH 10/45] Remove `actix_rt` & use standard tokio spawn (#3158) * Remove `actix_rt` & use standard tokio spawn * Adjust rust log back down * Format correctly * Update cargo lock * Add DB settings * Change name and update to latest rev * Clean up formatting changes * Move `worker_count` and `worker_retry_count` to settings * Update defaults * Use `0.4.4` instead of git branch --- Cargo.lock | 4 ++-- Cargo.toml | 2 +- config/defaults.hjson | 4 ++++ crates/api_common/Cargo.toml | 2 +- crates/api_common/src/request.rs | 2 +- crates/api_common/src/site.rs | 3 --- crates/api_crud/src/site/create.rs | 1 - crates/api_crud/src/site/update.rs | 1 - crates/apub/Cargo.toml | 2 +- crates/apub/src/collections/community_moderators.rs | 2 +- crates/apub/src/objects/comment.rs | 6 +++--- crates/apub/src/objects/community.rs | 2 +- crates/apub/src/objects/instance.rs | 2 +- crates/apub/src/objects/person.rs | 4 ++-- crates/apub/src/objects/post.rs | 2 +- crates/apub/src/objects/private_message.rs | 4 ++-- crates/db_schema/src/schema.rs | 1 - crates/db_schema/src/source/local_site.rs | 4 ---- crates/utils/src/settings/structs.rs | 6 ++++++ .../down.sql | 1 + .../up.sql | 1 + src/lib.rs | 13 ++++++------- src/main.rs | 2 +- 23 files changed, 36 insertions(+), 35 deletions(-) create mode 100644 migrations/2023-06-19-055530_add_retry_worker_setting/down.sql create mode 100644 migrations/2023-06-19-055530_add_retry_worker_setting/up.sql diff --git a/Cargo.lock b/Cargo.lock index 9590e8f13..9d575f578 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2543,7 +2543,6 @@ dependencies = [ name = "lemmy_api_common" version = "0.18.0" dependencies = [ - "actix-rt", "actix-web", "anyhow", "chrono", @@ -2561,6 +2560,7 @@ dependencies = [ "rosetta-i18n", "serde", "serde_with", + "tokio", "tracing", "ts-rs", "url", @@ -2592,7 +2592,6 @@ name = "lemmy_apub" version = "0.18.0" dependencies = [ "activitypub_federation", - "actix-rt", "actix-web", "anyhow", "assert-json-diff", @@ -2620,6 +2619,7 @@ dependencies = [ "sha2", "strum_macros", "task-local-extensions", + "tokio", "tracing", "url", "uuid", diff --git a/Cargo.toml b/Cargo.toml index 430deb082..07e41ab3b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,7 +89,7 @@ anyhow = "1.0.71" diesel_ltree = "0.3.0" typed-builder = "0.10.0" serial_test = "0.9.0" -tokio = "1.28.2" +tokio = { version = "1.28.2", features = ["full"] } sha2 = "0.10.6" regex = "1.8.4" once_cell = "1.18.0" diff --git a/config/defaults.hjson b/config/defaults.hjson index 4c38ddd45..6032f8fc9 100644 --- a/config/defaults.hjson +++ b/config/defaults.hjson @@ -76,4 +76,8 @@ port: 8536 # Whether the site is available over TLS. Needs to be true for federation to work. tls_enabled: true + # The number of activitypub federation workers that can be in-flight concurrently + worker_count: 0 + # The number of activitypub federation retry workers that can be in-flight concurrently + retry_count: 0 } diff --git a/crates/api_common/Cargo.toml b/crates/api_common/Cargo.toml index 46045d805..339d233a1 100644 --- a/crates/api_common/Cargo.toml +++ b/crates/api_common/Cargo.toml @@ -38,7 +38,7 @@ encoding = { version = "0.2.33", optional = true } anyhow = { workspace = true } futures = { workspace = true } uuid = { workspace = true } -actix-rt = { workspace = true } +tokio = { workspace = true } reqwest = { workspace = true } ts-rs = { workspace = true, optional = true } actix-web = { workspace = true } diff --git a/crates/api_common/src/request.rs b/crates/api_common/src/request.rs index c6f71b868..3139193a6 100644 --- a/crates/api_common/src/request.rs +++ b/crates/api_common/src/request.rs @@ -271,7 +271,7 @@ mod tests { use url::Url; // These helped with testing - #[actix_rt::test] + #[tokio::test] async fn test_site_metadata() { let settings = &SETTINGS.clone(); let client = reqwest::Client::builder() diff --git a/crates/api_common/src/site.rs b/crates/api_common/src/site.rs index 4d488ec1b..865acc0dc 100644 --- a/crates/api_common/src/site.rs +++ b/crates/api_common/src/site.rs @@ -177,7 +177,6 @@ pub struct CreateSite { pub rate_limit_search_per_second: Option, pub federation_enabled: Option, pub federation_debug: Option, - pub federation_worker_count: Option, pub captcha_enabled: Option, pub captcha_difficulty: Option, pub allowed_instances: Option>, @@ -250,8 +249,6 @@ pub struct EditSite { pub federation_enabled: Option, /// Enables federation debugging. pub federation_debug: Option, - /// The number of federation workers. - pub federation_worker_count: Option, /// Whether to enable captchas for signups. pub captcha_enabled: Option, /// The captcha difficulty. Can be easy, medium, or hard diff --git a/crates/api_crud/src/site/create.rs b/crates/api_crud/src/site/create.rs index 2a51309a4..a1669baef 100644 --- a/crates/api_crud/src/site/create.rs +++ b/crates/api_crud/src/site/create.rs @@ -122,7 +122,6 @@ impl PerformCrud for CreateSite { .slur_filter_regex(diesel_option_overwrite(&data.slur_filter_regex)) .actor_name_max_length(data.actor_name_max_length) .federation_enabled(data.federation_enabled) - .federation_worker_count(data.federation_worker_count) .captcha_enabled(data.captcha_enabled) .captcha_difficulty(data.captcha_difficulty.clone()) .build(); diff --git a/crates/api_crud/src/site/update.rs b/crates/api_crud/src/site/update.rs index fadde0a0b..6664d549a 100644 --- a/crates/api_crud/src/site/update.rs +++ b/crates/api_crud/src/site/update.rs @@ -123,7 +123,6 @@ impl PerformCrud for EditSite { .slur_filter_regex(diesel_option_overwrite(&data.slur_filter_regex)) .actor_name_max_length(data.actor_name_max_length) .federation_enabled(data.federation_enabled) - .federation_worker_count(data.federation_worker_count) .captcha_enabled(data.captcha_enabled) .captcha_difficulty(data.captcha_difficulty.clone()) .reports_email_admins(data.reports_email_admins) diff --git a/crates/apub/Cargo.toml b/crates/apub/Cargo.toml index 2007b541a..8570541f7 100644 --- a/crates/apub/Cargo.toml +++ b/crates/apub/Cargo.toml @@ -25,7 +25,7 @@ chrono = { workspace = true } serde_json = { workspace = true } serde = { workspace = true } actix-web = { workspace = true } -actix-rt = { workspace = true } +tokio = {workspace = true} tracing = { workspace = true } strum_macros = { workspace = true } url = { workspace = true } diff --git a/crates/apub/src/collections/community_moderators.rs b/crates/apub/src/collections/community_moderators.rs index c439da710..d53f86280 100644 --- a/crates/apub/src/collections/community_moderators.rs +++ b/crates/apub/src/collections/community_moderators.rs @@ -120,7 +120,7 @@ mod tests { }; use serial_test::serial; - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_community_moderators() { let context = init_context().await; diff --git a/crates/apub/src/objects/comment.rs b/crates/apub/src/objects/comment.rs index e2a03b8b3..16cb1542b 100644 --- a/crates/apub/src/objects/comment.rs +++ b/crates/apub/src/objects/comment.rs @@ -223,7 +223,7 @@ pub(crate) mod tests { LocalSite::delete(context.pool()).await.unwrap(); } - #[actix_rt::test] + #[tokio::test] #[serial] pub(crate) async fn test_parse_lemmy_comment() { let context = init_context().await; @@ -249,7 +249,7 @@ pub(crate) mod tests { cleanup(data, &context).await; } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_pleroma_comment() { let context = init_context().await; @@ -279,7 +279,7 @@ pub(crate) mod tests { cleanup(data, &context).await; } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_html_to_markdown_sanitize() { let parsed = parse_html("hello"); diff --git a/crates/apub/src/objects/community.rs b/crates/apub/src/objects/community.rs index 6526d2d26..888a7f458 100644 --- a/crates/apub/src/objects/community.rs +++ b/crates/apub/src/objects/community.rs @@ -242,7 +242,7 @@ pub(crate) mod tests { community } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_community() { let context = init_context().await; diff --git a/crates/apub/src/objects/instance.rs b/crates/apub/src/objects/instance.rs index 72d133441..6cd27fbbd 100644 --- a/crates/apub/src/objects/instance.rs +++ b/crates/apub/src/objects/instance.rs @@ -221,7 +221,7 @@ pub(crate) mod tests { site } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_instance() { let context = init_context().await; diff --git a/crates/apub/src/objects/person.rs b/crates/apub/src/objects/person.rs index c71d46ccf..3eeb733fd 100644 --- a/crates/apub/src/objects/person.rs +++ b/crates/apub/src/objects/person.rs @@ -223,7 +223,7 @@ pub(crate) mod tests { (person, site) } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_person() { let context = init_context().await; @@ -236,7 +236,7 @@ pub(crate) mod tests { cleanup((person, site), &context).await; } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_pleroma_person() { let context = init_context().await; diff --git a/crates/apub/src/objects/post.rs b/crates/apub/src/objects/post.rs index b255ffb9b..4ef9351ab 100644 --- a/crates/apub/src/objects/post.rs +++ b/crates/apub/src/objects/post.rs @@ -281,7 +281,7 @@ mod tests { use lemmy_db_schema::source::site::Site; use serial_test::serial; - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_post() { let context = init_context().await; diff --git a/crates/apub/src/objects/private_message.rs b/crates/apub/src/objects/private_message.rs index 01f576ff8..ae2637c58 100644 --- a/crates/apub/src/objects/private_message.rs +++ b/crates/apub/src/objects/private_message.rs @@ -187,7 +187,7 @@ mod tests { Site::delete(context.pool(), data.2.id).await.unwrap(); } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_lemmy_pm() { let context = init_context().await; @@ -213,7 +213,7 @@ mod tests { cleanup(data, &context).await; } - #[actix_rt::test] + #[tokio::test] #[serial] async fn test_parse_pleroma_pm() { let context = init_context().await; diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index ac4ddc47a..6714913f4 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -339,7 +339,6 @@ diesel::table! { slur_filter_regex -> Nullable, actor_name_max_length -> Int4, federation_enabled -> Bool, - federation_worker_count -> Int4, captcha_enabled -> Bool, #[max_length = 255] captcha_difficulty -> Varchar, diff --git a/crates/db_schema/src/source/local_site.rs b/crates/db_schema/src/source/local_site.rs index e65a61535..be93717a9 100644 --- a/crates/db_schema/src/source/local_site.rs +++ b/crates/db_schema/src/source/local_site.rs @@ -50,8 +50,6 @@ pub struct LocalSite { pub actor_name_max_length: i32, /// Whether federation is enabled. pub federation_enabled: bool, - /// The number of concurrent federation http workers. - pub federation_worker_count: i32, /// Whether captcha is enabled. pub captcha_enabled: bool, /// The captcha difficulty. @@ -85,7 +83,6 @@ pub struct LocalSiteInsertForm { pub slur_filter_regex: Option, pub actor_name_max_length: Option, pub federation_enabled: Option, - pub federation_worker_count: Option, pub captcha_enabled: Option, pub captcha_difficulty: Option, pub registration_mode: Option, @@ -112,7 +109,6 @@ pub struct LocalSiteUpdateForm { pub slur_filter_regex: Option>, pub actor_name_max_length: Option, pub federation_enabled: Option, - pub federation_worker_count: Option, pub captcha_enabled: Option, pub captcha_difficulty: Option, pub registration_mode: Option, diff --git a/crates/utils/src/settings/structs.rs b/crates/utils/src/settings/structs.rs index 6e200b224..5d0e642f6 100644 --- a/crates/utils/src/settings/structs.rs +++ b/crates/utils/src/settings/structs.rs @@ -39,6 +39,12 @@ pub struct Settings { #[default(None)] #[doku(skip)] pub opentelemetry_url: Option, + /// The number of activitypub federation workers that can be in-flight concurrently + #[default(0)] + pub worker_count: usize, + /// The number of activitypub federation retry workers that can be in-flight concurrently + #[default(0)] + pub retry_count: usize, } #[derive(Debug, Deserialize, Serialize, Clone, SmartDefault, Document)] diff --git a/migrations/2023-06-19-055530_add_retry_worker_setting/down.sql b/migrations/2023-06-19-055530_add_retry_worker_setting/down.sql new file mode 100644 index 000000000..e3c200a15 --- /dev/null +++ b/migrations/2023-06-19-055530_add_retry_worker_setting/down.sql @@ -0,0 +1 @@ +alter table local_site add column federation_worker_count int default 64 not null; \ No newline at end of file diff --git a/migrations/2023-06-19-055530_add_retry_worker_setting/up.sql b/migrations/2023-06-19-055530_add_retry_worker_setting/up.sql new file mode 100644 index 000000000..2aac86f85 --- /dev/null +++ b/migrations/2023-06-19-055530_add_retry_worker_setting/up.sql @@ -0,0 +1 @@ +alter table local_site drop column federation_worker_count; \ No newline at end of file diff --git a/src/lib.rs b/src/lib.rs index 86cf400b6..d919acc05 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -139,24 +139,23 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> { }); } + let settings_bind = settings.clone(); + let federation_config = FederationConfig::builder() .domain(settings.hostname.clone()) .app_data(context.clone()) .client(client.clone()) .http_fetch_limit(FEDERATION_HTTP_FETCH_LIMIT) - .worker_count(local_site.federation_worker_count as usize) + .worker_count(settings.worker_count) + .retry_count(settings.retry_count) .debug(cfg!(debug_assertions)) .http_signature_compat(true) .url_verifier(Box::new(VerifyUrlData(context.pool().clone()))) .build() - .await - .expect("configure federation"); + .await?; // Create Http server with websocket support - let settings_bind = settings.clone(); HttpServer::new(move || { - let context = context.clone(); - let cors_config = if cfg!(debug_assertions) { Cors::permissive() } else { @@ -173,7 +172,7 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> { )) .wrap(cors_config) .wrap(TracingLogger::::new()) - .app_data(Data::new(context)) + .app_data(Data::new(context.clone())) .app_data(Data::new(rate_limit_cell.clone())) .wrap(FederationMiddleware::new(federation_config.clone())) // The routes diff --git a/src/main.rs b/src/main.rs index 315fe84be..5fc03ed02 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,7 +1,7 @@ use lemmy_server::{init_logging, start_lemmy_server}; use lemmy_utils::{error::LemmyError, settings::SETTINGS}; -#[actix_web::main] +#[tokio::main] pub async fn main() -> Result<(), LemmyError> { init_logging(&SETTINGS.opentelemetry_url)?; #[cfg(not(feature = "embed-pictrs"))] From 6b28f8c616593be2c6905a39e98e1df2de956fe2 Mon Sep 17 00:00:00 2001 From: Sander Saarend Date: Mon, 26 Jun 2023 11:25:38 +0300 Subject: [PATCH 11/45] Add support for sslmode=require for diesel-async DB connections (#3189) --- Cargo.lock | 92 +++++++++++++++++++++++++++++------ Cargo.toml | 9 ++++ crates/db_schema/Cargo.toml | 6 ++- crates/db_schema/src/utils.rs | 64 ++++++++++++++++++++++-- 4 files changed, 152 insertions(+), 19 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d575f578..99d94948f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -215,7 +215,7 @@ dependencies = [ "futures-util", "mio", "num_cpus", - "socket2", + "socket2 0.4.9", "tokio", "tracing", ] @@ -245,7 +245,7 @@ dependencies = [ "http", "log", "pin-project-lite", - "tokio-rustls", + "tokio-rustls 0.23.4", "tokio-util 0.7.4", "webpki-roots", ] @@ -297,7 +297,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2", + "socket2 0.4.9", "time 0.3.15", "url", ] @@ -496,7 +496,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "rand 0.8.5", - "rustls", + "rustls 0.20.7", "serde", "serde_json", "serde_urlencoded", @@ -2262,7 +2262,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -2640,9 +2640,11 @@ dependencies = [ "diesel-derive-newtype", "diesel_ltree", "diesel_migrations", + "futures-util", "lemmy_utils", "once_cell", "regex", + "rustls 0.21.2", "serde", "serde_json", "serde_with", @@ -2651,6 +2653,8 @@ dependencies = [ "strum", "strum_macros", "tokio", + "tokio-postgres", + "tokio-postgres-rustls", "tracing", "ts-rs", "typed-builder", @@ -2736,6 +2740,7 @@ dependencies = [ "diesel", "diesel-async", "doku", + "futures-util", "lemmy_api", "lemmy_api_common", "lemmy_api_crud", @@ -2749,9 +2754,12 @@ dependencies = [ "reqwest", "reqwest-middleware", "reqwest-tracing", + "rustls 0.21.2", "serde", "serde_json", "tokio", + "tokio-postgres", + "tokio-postgres-rustls", "tracing", "tracing-actix-web 0.6.2", "tracing-error", @@ -2820,7 +2828,7 @@ dependencies = [ "nom 7.1.1", "once_cell", "quoted_printable", - "socket2", + "socket2 0.4.9", ] [[package]] @@ -3932,11 +3940,11 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.6.4" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "878c6cbf956e03af9aa8204b407b9cbf47c072164800aa918c516cd4b056c50c" +checksum = "78b7fa9f396f51dffd61546fd8573ee20592287996568e6175ceb0f8699ad75d" dependencies = [ - "base64 0.13.1", + "base64 0.21.2", "byteorder", "bytes", "fallible-iterator", @@ -4495,6 +4503,28 @@ dependencies = [ "webpki", ] +[[package]] +name = "rustls" +version = "0.21.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e32ca28af694bc1bbf399c33a516dbdf1c90090b8ab23c2bc24f834aa2247f5f" +dependencies = [ + "log", + "ring", + "rustls-webpki", + "sct", +] + +[[package]] +name = "rustls-webpki" +version = "0.100.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +dependencies = [ + "ring", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.9" @@ -4859,6 +4889,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "spin" version = "0.5.2" @@ -5304,7 +5344,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.4.9", "tokio-macros", "tracing", "windows-sys 0.48.0", @@ -5343,9 +5383,9 @@ dependencies = [ [[package]] name = "tokio-postgres" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "29a12c1b3e0704ae7dfc25562629798b29c72e6b1d0a681b6f29ab4ae5e7f7bf" +checksum = "6e89f6234aa8fd43779746012fcf53603cdb91fdd8399aa0de868c2d56b6dde1" dependencies = [ "async-trait", "byteorder", @@ -5360,22 +5400,46 @@ dependencies = [ "pin-project-lite", "postgres-protocol", "postgres-types", - "socket2", + "socket2 0.5.3", "tokio", "tokio-util 0.7.4", ] +[[package]] +name = "tokio-postgres-rustls" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd5831152cb0d3f79ef5523b357319ba154795d64c7078b2daa95a803b54057f" +dependencies = [ + "futures", + "ring", + "rustls 0.21.2", + "tokio", + "tokio-postgres", + "tokio-rustls 0.24.1", +] + [[package]] name = "tokio-rustls" version = "0.23.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c43ee83903113e03984cb9e5cebe6c04a5116269e900e3ddba8f068a62adda59" dependencies = [ - "rustls", + "rustls 0.20.7", "tokio", "webpki", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.2", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.11" diff --git a/Cargo.toml b/Cargo.toml index 07e41ab3b..2ee5a5308 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -107,6 +107,10 @@ rand = "0.8.5" opentelemetry = { version = "0.17.0", features = ["rt-tokio"] } tracing-opentelemetry = { version = "0.17.4" } ts-rs = { version = "6.2", features = ["serde-compat", "format", "chrono-impl"] } +rustls = { version ="0.21.2", features = ["dangerous_configuration"]} +futures-util = "0.3.28" +tokio-postgres = "0.7.8" +tokio-postgres-rustls = "0.10.0" [dependencies] lemmy_api = { workspace = true } @@ -140,3 +144,8 @@ opentelemetry-otlp = { version = "0.10.0", optional = true } pict-rs = { version = "0.4.0-rc.3", optional = true } tokio.workspace = true actix-cors = "0.6.4" +rustls = { workspace = true } +futures-util = { workspace = true } +tokio-postgres = { workspace = true } +tokio-postgres-rustls = { workspace = true } + diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index aa26382c0..6c89ab9fa 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -43,7 +43,11 @@ async-trait = { workspace = true } tokio = { workspace = true } tracing = { workspace = true } deadpool = { version = "0.9.5", features = ["rt_tokio_1"], optional = true } -ts-rs = { workspace = true, optional = true } +ts-rs = { workspace = true, optional = true } +rustls = { workspace = true } +futures-util = { workspace = true } +tokio-postgres = { workspace = true } +tokio-postgres-rustls = { workspace = true } [dev-dependencies] serial_test = { workspace = true } diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index 9954f623b..7ba5d5e7b 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -12,7 +12,7 @@ use diesel::{ backend::Backend, deserialize::FromSql, pg::Pg, - result::{Error as DieselError, Error::QueryBuilderError}, + result::{ConnectionError, ConnectionResult, Error as DieselError, Error::QueryBuilderError}, serialize::{Output, ToSql}, sql_types::Text, PgConnection, @@ -25,11 +25,21 @@ use diesel_async::{ }, }; use diesel_migrations::EmbeddedMigrations; +use futures_util::{future::BoxFuture, FutureExt}; use lemmy_utils::{error::LemmyError, settings::structs::Settings}; use once_cell::sync::Lazy; use regex::Regex; -use std::{env, env::VarError, time::Duration}; -use tracing::info; +use rustls::{ + client::{ServerCertVerified, ServerCertVerifier}, + ServerName, +}; +use std::{ + env, + env::VarError, + sync::Arc, + time::{Duration, SystemTime}, +}; +use tracing::{error, info}; use url::Url; const FETCH_LIMIT_DEFAULT: i64 = 10; @@ -136,7 +146,15 @@ pub fn diesel_option_overwrite_to_url_create( async fn build_db_pool_settings_opt(settings: Option<&Settings>) -> Result { let db_url = get_database_url(settings); let pool_size = settings.map(|s| s.database.pool_size).unwrap_or(5); - let manager = AsyncDieselConnectionManager::::new(&db_url); + // We only support TLS with sslmode=require currently + let tls_enabled = db_url.contains("sslmode=require"); + let manager = if tls_enabled { + // diesel-async does not support any TLS connections out of the box, so we need to manually + // provide a setup function which handles creating the connection + AsyncDieselConnectionManager::::new_with_setup(&db_url, establish_connection) + } else { + AsyncDieselConnectionManager::::new(&db_url) + }; let pool = Pool::builder(manager) .max_size(pool_size) .wait_timeout(POOL_TIMEOUT) @@ -153,6 +171,44 @@ async fn build_db_pool_settings_opt(settings: Option<&Settings>) -> Result BoxFuture> { + let fut = async { + let rustls_config = rustls::ClientConfig::builder() + .with_safe_defaults() + .with_custom_certificate_verifier(Arc::new(NoCertVerifier {})) + .with_no_client_auth(); + + let tls = tokio_postgres_rustls::MakeRustlsConnect::new(rustls_config); + let (client, conn) = tokio_postgres::connect(config, tls) + .await + .map_err(|e| ConnectionError::BadConnection(e.to_string()))?; + tokio::spawn(async move { + if let Err(e) = conn.await { + error!("Database connection failed: {e}"); + } + }); + AsyncPgConnection::try_from(client).await + }; + fut.boxed() +} + +struct NoCertVerifier {} + +impl ServerCertVerifier for NoCertVerifier { + fn verify_server_cert( + &self, + _end_entity: &rustls::Certificate, + _intermediates: &[rustls::Certificate], + _server_name: &ServerName, + _scts: &mut dyn Iterator, + _ocsp_response: &[u8], + _now: SystemTime, + ) -> Result { + // Will verify all (even invalid) certs without any checks (sslmode=require) + Ok(ServerCertVerified::assertion()) + } +} + pub const MIGRATIONS: EmbeddedMigrations = embed_migrations!(); pub fn run_migrations(db_url: &str) { From ce0cf0e41bf00d5b741f0cb4789c508df72edd79 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 26 Jun 2023 10:45:37 +0200 Subject: [PATCH 12/45] Remove DELETED_REPLACEMENT_URL --- crates/db_schema/src/impls/post.rs | 11 ++--------- crates/db_schema/src/utils.rs | 1 - 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/crates/db_schema/src/impls/post.rs b/crates/db_schema/src/impls/post.rs index 7f59d29ec..6a4d53d3a 100644 --- a/crates/db_schema/src/impls/post.rs +++ b/crates/db_schema/src/impls/post.rs @@ -27,14 +27,7 @@ use crate::{ PostUpdateForm, }, traits::{Crud, Likeable, Readable, Saveable}, - utils::{ - get_conn, - naive_now, - DbPool, - DELETED_REPLACEMENT_TEXT, - DELETED_REPLACEMENT_URL, - FETCH_LIMIT_MAX, - }, + utils::{get_conn, naive_now, DbPool, DELETED_REPLACEMENT_TEXT, FETCH_LIMIT_MAX}, }; use ::url::Url; use diesel::{dsl::insert_into, result::Error, ExpressionMethods, QueryDsl, TextExpressionMethods}; @@ -121,7 +114,7 @@ impl Post { diesel::update(post.filter(creator_id.eq(for_creator_id))) .set(( name.eq(DELETED_REPLACEMENT_TEXT), - url.eq(DELETED_REPLACEMENT_URL), + url.eq(Option::<&str>::None), body.eq(DELETED_REPLACEMENT_TEXT), deleted.eq(true), updated.eq(naive_now()), diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index 7ba5d5e7b..fdd445637 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -279,7 +279,6 @@ pub mod functions { } pub const DELETED_REPLACEMENT_TEXT: &str = "*Permanently Deleted*"; -pub const DELETED_REPLACEMENT_URL: &str = "https://join-lemmy.org/"; impl ToSql for DbUrl { fn to_sql(&self, out: &mut Output) -> diesel::serialize::Result { From 203e35899e12e8696d0c871b5c12b27a5751b4a2 Mon Sep 17 00:00:00 2001 From: Neshura Date: Mon, 26 Jun 2023 10:47:01 +0200 Subject: [PATCH 13/45] Add separate Post check for is_valid_body_field (#3263) * Add separate Post check for is_valid_body_field * Modify is_valid_body_check for posts only * Fix check var reinit in validation.rs * Extra empty line to rerun woodpecker with changes * Change Option to bool, add false to non-post calls * Woodpecker trick.. again * Probable rust_fmt fail fixed * cargo_clippy changes * Missing space between = and if * Remove ; after body length checks --- crates/api/src/community/ban.rs | 2 +- crates/api/src/local_user/ban_person.rs | 2 +- crates/api_crud/src/comment/create.rs | 2 +- crates/api_crud/src/comment/update.rs | 2 +- crates/api_crud/src/community/create.rs | 2 +- crates/api_crud/src/community/update.rs | 2 +- crates/api_crud/src/post/create.rs | 2 +- crates/api_crud/src/post/update.rs | 2 +- crates/api_crud/src/private_message/create.rs | 2 +- crates/api_crud/src/private_message/update.rs | 2 +- crates/api_crud/src/site/create.rs | 2 +- crates/api_crud/src/site/update.rs | 2 +- crates/utils/src/utils/validation.rs | 10 ++++++++-- 13 files changed, 20 insertions(+), 14 deletions(-) diff --git a/crates/api/src/community/ban.rs b/crates/api/src/community/ban.rs index a0fd7bf18..330c2c56d 100644 --- a/crates/api/src/community/ban.rs +++ b/crates/api/src/community/ban.rs @@ -42,7 +42,7 @@ impl Perform for BanFromCommunity { // Verify that only mods or admins can ban is_mod_or_admin(context.pool(), local_user_view.person.id, community_id).await?; - is_valid_body_field(&data.reason)?; + is_valid_body_field(&data.reason, false)?; let community_user_ban_form = CommunityPersonBanForm { community_id: data.community_id, diff --git a/crates/api/src/local_user/ban_person.rs b/crates/api/src/local_user/ban_person.rs index 452557d2c..2c2d363e3 100644 --- a/crates/api/src/local_user/ban_person.rs +++ b/crates/api/src/local_user/ban_person.rs @@ -30,7 +30,7 @@ impl Perform for BanPerson { // Make sure user is an admin is_admin(&local_user_view)?; - is_valid_body_field(&data.reason)?; + is_valid_body_field(&data.reason, false)?; let ban = data.ban; let banned_person_id = data.person_id; diff --git a/crates/api_crud/src/comment/create.rs b/crates/api_crud/src/comment/create.rs index 4ef8686e2..b3b1efecd 100644 --- a/crates/api_crud/src/comment/create.rs +++ b/crates/api_crud/src/comment/create.rs @@ -49,7 +49,7 @@ impl PerformCrud for CreateComment { &data.content.clone(), &local_site_to_slur_regex(&local_site), ); - is_valid_body_field(&Some(content_slurs_removed.clone()))?; + is_valid_body_field(&Some(content_slurs_removed.clone()), false)?; // Check for a community ban let post_id = data.post_id; diff --git a/crates/api_crud/src/comment/update.rs b/crates/api_crud/src/comment/update.rs index 86bdb52e2..3504e784d 100644 --- a/crates/api_crud/src/comment/update.rs +++ b/crates/api_crud/src/comment/update.rs @@ -64,7 +64,7 @@ impl PerformCrud for EditComment { .as_ref() .map(|c| remove_slurs(c, &local_site_to_slur_regex(&local_site))); - is_valid_body_field(&content_slurs_removed)?; + is_valid_body_field(&content_slurs_removed, false)?; let comment_id = data.comment_id; let form = CommentUpdateForm::builder() diff --git a/crates/api_crud/src/community/create.rs b/crates/api_crud/src/community/create.rs index 850e9f2f5..0e55beac9 100644 --- a/crates/api_crud/src/community/create.rs +++ b/crates/api_crud/src/community/create.rs @@ -67,7 +67,7 @@ impl PerformCrud for CreateCommunity { check_slurs_opt(&data.description, &slur_regex)?; is_valid_actor_name(&data.name, local_site.actor_name_max_length as usize)?; - is_valid_body_field(&data.description)?; + is_valid_body_field(&data.description, false)?; // Double check for duplicate community actor_ids let community_actor_id = generate_local_apub_endpoint( diff --git a/crates/api_crud/src/community/update.rs b/crates/api_crud/src/community/update.rs index 7494cd342..dec62865f 100644 --- a/crates/api_crud/src/community/update.rs +++ b/crates/api_crud/src/community/update.rs @@ -39,7 +39,7 @@ impl PerformCrud for EditCommunity { let slur_regex = local_site_to_slur_regex(&local_site); check_slurs_opt(&data.title, &slur_regex)?; check_slurs_opt(&data.description, &slur_regex)?; - is_valid_body_field(&data.description)?; + is_valid_body_field(&data.description, false)?; // Verify its a mod (only mods can edit it) let community_id = data.community_id; diff --git a/crates/api_crud/src/post/create.rs b/crates/api_crud/src/post/create.rs index cd2cf1c3d..8ff1b678a 100644 --- a/crates/api_crud/src/post/create.rs +++ b/crates/api_crud/src/post/create.rs @@ -57,7 +57,7 @@ impl PerformCrud for CreatePost { let url = data_url.map(clean_url_params).map(Into::into); // TODO no good way to handle a "clear" is_valid_post_title(&data.name)?; - is_valid_body_field(&data.body)?; + is_valid_body_field(&data.body, true)?; check_community_ban(local_user_view.person.id, data.community_id, context.pool()).await?; check_community_deleted_or_removed(data.community_id, context.pool()).await?; diff --git a/crates/api_crud/src/post/update.rs b/crates/api_crud/src/post/update.rs index af2c63c50..a540f454f 100644 --- a/crates/api_crud/src/post/update.rs +++ b/crates/api_crud/src/post/update.rs @@ -49,7 +49,7 @@ impl PerformCrud for EditPost { is_valid_post_title(name)?; } - is_valid_body_field(&data.body)?; + is_valid_body_field(&data.body, true)?; let post_id = data.post_id; let orig_post = Post::read(context.pool(), post_id).await?; diff --git a/crates/api_crud/src/private_message/create.rs b/crates/api_crud/src/private_message/create.rs index 3f1d4ef89..e1a855463 100644 --- a/crates/api_crud/src/private_message/create.rs +++ b/crates/api_crud/src/private_message/create.rs @@ -43,7 +43,7 @@ impl PerformCrud for CreatePrivateMessage { &data.content.clone(), &local_site_to_slur_regex(&local_site), ); - is_valid_body_field(&Some(content_slurs_removed.clone()))?; + is_valid_body_field(&Some(content_slurs_removed.clone()), false)?; check_person_block(local_user_view.person.id, data.recipient_id, context.pool()).await?; diff --git a/crates/api_crud/src/private_message/update.rs b/crates/api_crud/src/private_message/update.rs index cc3c377b8..b2d8e48f9 100644 --- a/crates/api_crud/src/private_message/update.rs +++ b/crates/api_crud/src/private_message/update.rs @@ -41,7 +41,7 @@ impl PerformCrud for EditPrivateMessage { // Doing the update let content_slurs_removed = remove_slurs(&data.content, &local_site_to_slur_regex(&local_site)); - is_valid_body_field(&Some(content_slurs_removed.clone()))?; + is_valid_body_field(&Some(content_slurs_removed.clone()), false)?; let private_message_id = data.private_message_id; PrivateMessage::update( diff --git a/crates/api_crud/src/site/create.rs b/crates/api_crud/src/site/create.rs index a1669baef..e7486e63a 100644 --- a/crates/api_crud/src/site/create.rs +++ b/crates/api_crud/src/site/create.rs @@ -73,7 +73,7 @@ impl PerformCrud for CreateSite { site_description_length_check(desc)?; } - is_valid_body_field(&data.sidebar)?; + is_valid_body_field(&data.sidebar, false)?; let application_question = diesel_option_overwrite(&data.application_question); check_application_question( diff --git a/crates/api_crud/src/site/update.rs b/crates/api_crud/src/site/update.rs index 6664d549a..fa800a5a9 100644 --- a/crates/api_crud/src/site/update.rs +++ b/crates/api_crud/src/site/update.rs @@ -67,7 +67,7 @@ impl PerformCrud for EditSite { site_description_length_check(desc)?; } - is_valid_body_field(&data.sidebar)?; + is_valid_body_field(&data.sidebar, false)?; let application_question = diesel_option_overwrite(&data.application_question); check_application_question( diff --git a/crates/utils/src/utils/validation.rs b/crates/utils/src/utils/validation.rs index 41103332c..621543b47 100644 --- a/crates/utils/src/utils/validation.rs +++ b/crates/utils/src/utils/validation.rs @@ -18,6 +18,7 @@ static CLEAN_URL_PARAMS_REGEX: Lazy = Lazy::new(|| { .expect("compile regex") }); const BODY_MAX_LENGTH: usize = 10000; +const POST_BODY_MAX_LENGTH: usize = 50000; const BIO_MAX_LENGTH: usize = 300; fn has_newline(name: &str) -> bool { @@ -68,9 +69,14 @@ pub fn is_valid_post_title(title: &str) -> LemmyResult<()> { } /// This could be post bodies, comments, or any description field -pub fn is_valid_body_field(body: &Option) -> LemmyResult<()> { +pub fn is_valid_body_field(body: &Option, post: bool) -> LemmyResult<()> { if let Some(body) = body { - let check = body.chars().count() <= BODY_MAX_LENGTH; + let check = if post { + body.chars().count() <= POST_BODY_MAX_LENGTH + } else { + body.chars().count() <= BODY_MAX_LENGTH + }; + if !check { Err(LemmyError::from_message("invalid_body_field")) } else { From 21d53497851e367a72e1ac41525820452fae41b8 Mon Sep 17 00:00:00 2001 From: TKilFree Date: Mon, 26 Jun 2023 09:47:39 +0100 Subject: [PATCH 14/45] feat: allow all admins to purge content (#3271) --- crates/api/src/site/purge/comment.rs | 6 +++--- crates/api/src/site/purge/community.rs | 6 +++--- crates/api/src/site/purge/person.rs | 6 +++--- crates/api/src/site/purge/post.rs | 6 +++--- crates/api_common/src/utils.rs | 13 ------------- 5 files changed, 12 insertions(+), 25 deletions(-) diff --git a/crates/api/src/site/purge/comment.rs b/crates/api/src/site/purge/comment.rs index c8abf57d9..7beba9c0b 100644 --- a/crates/api/src/site/purge/comment.rs +++ b/crates/api/src/site/purge/comment.rs @@ -3,7 +3,7 @@ use actix_web::web::Data; use lemmy_api_common::{ context::LemmyContext, site::{PurgeComment, PurgeItemResponse}, - utils::{is_top_admin, local_user_view_from_jwt}, + utils::{is_admin, local_user_view_from_jwt}, }; use lemmy_db_schema::{ source::{ @@ -23,8 +23,8 @@ impl Perform for PurgeComment { let data: &Self = self; let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; - // Only let the top admin purge an item - is_top_admin(context.pool(), local_user_view.person.id).await?; + // Only let admin purge an item + is_admin(&local_user_view)?; let comment_id = data.comment_id; diff --git a/crates/api/src/site/purge/community.rs b/crates/api/src/site/purge/community.rs index 111c437ec..50482b73a 100644 --- a/crates/api/src/site/purge/community.rs +++ b/crates/api/src/site/purge/community.rs @@ -4,7 +4,7 @@ use lemmy_api_common::{ context::LemmyContext, request::purge_image_from_pictrs, site::{PurgeCommunity, PurgeItemResponse}, - utils::{is_top_admin, local_user_view_from_jwt, purge_image_posts_for_community}, + utils::{is_admin, local_user_view_from_jwt, purge_image_posts_for_community}, }; use lemmy_db_schema::{ source::{ @@ -24,8 +24,8 @@ impl Perform for PurgeCommunity { let data: &Self = self; let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; - // Only let the top admin purge an item - is_top_admin(context.pool(), local_user_view.person.id).await?; + // Only let admin purge an item + is_admin(&local_user_view)?; let community_id = data.community_id; diff --git a/crates/api/src/site/purge/person.rs b/crates/api/src/site/purge/person.rs index 5273110ed..f0cbc7e8d 100644 --- a/crates/api/src/site/purge/person.rs +++ b/crates/api/src/site/purge/person.rs @@ -4,7 +4,7 @@ use lemmy_api_common::{ context::LemmyContext, request::purge_image_from_pictrs, site::{PurgeItemResponse, PurgePerson}, - utils::{is_top_admin, local_user_view_from_jwt, purge_image_posts_for_person}, + utils::{is_admin, local_user_view_from_jwt, purge_image_posts_for_person}, }; use lemmy_db_schema::{ source::{ @@ -24,8 +24,8 @@ impl Perform for PurgePerson { let data: &Self = self; let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; - // Only let the top admin purge an item - is_top_admin(context.pool(), local_user_view.person.id).await?; + // Only let admin purge an item + is_admin(&local_user_view)?; // Read the person to get their images let person_id = data.person_id; diff --git a/crates/api/src/site/purge/post.rs b/crates/api/src/site/purge/post.rs index 01d0332eb..65d390f8e 100644 --- a/crates/api/src/site/purge/post.rs +++ b/crates/api/src/site/purge/post.rs @@ -4,7 +4,7 @@ use lemmy_api_common::{ context::LemmyContext, request::purge_image_from_pictrs, site::{PurgeItemResponse, PurgePost}, - utils::{is_top_admin, local_user_view_from_jwt}, + utils::{is_admin, local_user_view_from_jwt}, }; use lemmy_db_schema::{ source::{ @@ -24,8 +24,8 @@ impl Perform for PurgePost { let data: &Self = self; let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; - // Only let the top admin purge an item - is_top_admin(context.pool(), local_user_view.person.id).await?; + // Only let admin purge an item + is_admin(&local_user_view)?; let post_id = data.post_id; diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index 455acc182..4781ce923 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -32,7 +32,6 @@ use lemmy_db_views_actor::structs::{ CommunityModeratorView, CommunityPersonBanView, CommunityView, - PersonView, }; use lemmy_utils::{ claims::Claims, @@ -79,18 +78,6 @@ pub async fn is_mod_or_admin_opt( } } -pub async fn is_top_admin(pool: &DbPool, person_id: PersonId) -> Result<(), LemmyError> { - let admins = PersonView::admins(pool).await?; - let top_admin = admins - .first() - .ok_or_else(|| LemmyError::from_message("no admins"))?; - - if top_admin.person.id != person_id { - return Err(LemmyError::from_message("not_top_admin")); - } - Ok(()) -} - pub fn is_admin(local_user_view: &LocalUserView) -> Result<(), LemmyError> { if !local_user_view.person.admin { return Err(LemmyError::from_message("not_an_admin")); From c5886404efaba6ba477e6c30830115473f497706 Mon Sep 17 00:00:00 2001 From: Scott <97430840+scme0@users.noreply.github.com> Date: Mon, 26 Jun 2023 18:23:04 +0930 Subject: [PATCH 15/45] Update DB local_user.theme type to text (#3266) * Update local_user.theme type to text * fix default value * Undo auto generate changes --- crates/db_schema/src/schema.rs | 3 +-- .../2023-06-22-101245_increase_user_theme_column_size/down.sql | 2 ++ .../2023-06-22-101245_increase_user_theme_column_size/up.sql | 2 ++ 3 files changed, 5 insertions(+), 2 deletions(-) create mode 100644 migrations/2023-06-22-101245_increase_user_theme_column_size/down.sql create mode 100644 migrations/2023-06-22-101245_increase_user_theme_column_size/up.sql diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index 6714913f4..aef6fbe5d 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -381,8 +381,7 @@ diesel::table! { password_encrypted -> Text, email -> Nullable, show_nsfw -> Bool, - #[max_length = 20] - theme -> Varchar, + theme -> Text, default_sort_type -> SortTypeEnum, default_listing_type -> ListingTypeEnum, #[max_length = 20] diff --git a/migrations/2023-06-22-101245_increase_user_theme_column_size/down.sql b/migrations/2023-06-22-101245_increase_user_theme_column_size/down.sql new file mode 100644 index 000000000..0731e0682 --- /dev/null +++ b/migrations/2023-06-22-101245_increase_user_theme_column_size/down.sql @@ -0,0 +1,2 @@ +alter table only local_user alter column theme TYPE character varying(20); +alter table only local_user alter column theme set default 'browser'::character varying; \ No newline at end of file diff --git a/migrations/2023-06-22-101245_increase_user_theme_column_size/up.sql b/migrations/2023-06-22-101245_increase_user_theme_column_size/up.sql new file mode 100644 index 000000000..cbab25663 --- /dev/null +++ b/migrations/2023-06-22-101245_increase_user_theme_column_size/up.sql @@ -0,0 +1,2 @@ +alter table only local_user alter column theme type text; +alter table only local_user alter column theme set default 'browser'::text; From ddfa112e0b48d8facdefb1f09fb5f1db6c2c6677 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Mon, 26 Jun 2023 11:18:29 +0200 Subject: [PATCH 16/45] Allow wildcard imports in schema.rs (#3293) Dont mess with auto-generated code, this avoids problems with clippy after running diesel commands --- crates/db_schema/src/lib.rs | 1 + crates/db_schema/src/schema.rs | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/crates/db_schema/src/lib.rs b/crates/db_schema/src/lib.rs index 4ab26981b..04ec4e7da 100644 --- a/crates/db_schema/src/lib.rs +++ b/crates/db_schema/src/lib.rs @@ -26,6 +26,7 @@ pub mod impls; pub mod newtypes; #[cfg(feature = "full")] #[rustfmt::skip] +#[allow(clippy::wildcard_imports)] pub mod schema; pub mod source; #[cfg(feature = "full")] diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index aef6fbe5d..abd3ca22a 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -65,7 +65,7 @@ diesel::table! { } diesel::table! { - use diesel::sql_types::{Bool, Int4, Nullable, Text, Timestamp, Varchar}; + use diesel::sql_types::*; use diesel_ltree::sql_types::Ltree; comment (id) { @@ -317,7 +317,7 @@ diesel::table! { } diesel::table! { - use diesel::sql_types::{Bool, Int4, Nullable, Text, Timestamp, Varchar}; + use diesel::sql_types::*; use super::sql_types::ListingTypeEnum; use super::sql_types::RegistrationModeEnum; @@ -371,7 +371,7 @@ diesel::table! { } diesel::table! { - use diesel::sql_types::{Bool, Int4, Nullable, Text, Timestamp, Varchar}; + use diesel::sql_types::*; use super::sql_types::SortTypeEnum; use super::sql_types::ListingTypeEnum; From 2f3d60a63b3d88ec78eec75b6f6ec740ebb9f15c Mon Sep 17 00:00:00 2001 From: Peter Dave Hello Date: Mon, 26 Jun 2023 17:59:02 +0800 Subject: [PATCH 17/45] Leave no apk cache in Docker image (#3327) --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 5943e1710..010527515 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -29,7 +29,7 @@ RUN \ FROM alpine:3 as lemmy # Install libpq for postgres -RUN apk add libpq +RUN apk add --no-cache libpq # Copy resources COPY --from=builder /app/lemmy_server /app/lemmy From 01dc1efe775adce5f7710172b2ed579ce64115d7 Mon Sep 17 00:00:00 2001 From: anonion Date: Mon, 26 Jun 2023 04:07:22 -0600 Subject: [PATCH 18/45] Fix lemmy UI environment variable (#3299) --- docker/docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 2033ee8c7..00059cd15 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -74,7 +74,7 @@ services: - LEMMY_UI_LEMMY_INTERNAL_HOST=lemmy:8536 # set the outside hostname here - LEMMY_UI_LEMMY_EXTERNAL_HOST=localhost:1236 - - LEMMY_HTTPS=false + - LEMMY_UI_HTTPS=false - LEMMY_UI_DEBUG=true depends_on: - lemmy From 3d7d6b253086f1ac78e6dd459bc4c904df45dbfa Mon Sep 17 00:00:00 2001 From: perillamint Date: Mon, 26 Jun 2023 19:10:04 +0900 Subject: [PATCH 19/45] Respond with `Content-Type: application/activity+json` (#3353) As per ActivityPub specification, the return type should be `application/activity+json`, not `application/json`. --- crates/apub/src/http/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/apub/src/http/mod.rs b/crates/apub/src/http/mod.rs index 0b8c8f153..947a56adc 100644 --- a/crates/apub/src/http/mod.rs +++ b/crates/apub/src/http/mod.rs @@ -48,7 +48,7 @@ where Ok( HttpResponse::Ok() .content_type(FEDERATION_CONTENT_TYPE) - .content_type("application/json") + .content_type("application/activity+json") .body(json), ) } @@ -61,7 +61,7 @@ fn create_apub_tombstone_response>(id: T) -> LemmyResult Date: Mon, 26 Jun 2023 13:12:38 +0300 Subject: [PATCH 20/45] Add Liftoff to README.md (#3357) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index f759c1cde..240bde516 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,7 @@ Each Lemmy server can set its own moderation policy; appointing site-wide admins - [Jerboa - A native Android app made by Lemmy's developers](https://github.com/dessalines/jerboa) - [Mlem - A Lemmy client for iOS](https://github.com/buresdv/Mlem) - [Lemoa - A Gtk client for Lemmy on Linux](https://github.com/lemmy-gtk/lemoa) +- [Liftoff - A Lemmy for Windows , Linux and Android ](https://github.com/liftoff-app/liftoff) ### Libraries From a39e948b525397416378d7aab0ca4d25cc9be423 Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 26 Jun 2023 12:14:27 +0200 Subject: [PATCH 21/45] Remove unused actix_rt dependency --- Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Cargo.toml b/Cargo.toml index 2ee5a5308..c2cbc4879 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -100,7 +100,6 @@ strum_macros = "0.24.3" itertools = "0.10.5" futures = "0.3.28" http = "0.2.9" -actix-rt = { version = "2.8.0", default-features = false } percent-encoding = "2.3.0" rosetta-i18n = "0.1.2" rand = "0.8.5" From 9406c3ad2bc0cd265b766632d143945bc3c93989 Mon Sep 17 00:00:00 2001 From: cetra3 Date: Mon, 26 Jun 2023 19:53:21 +0930 Subject: [PATCH 22/45] Adjust the config check to be a separate faster to compile binary (#3313) --- crates/utils/src/main.rs | 16 ++++++++++++++++ scripts/update_config_defaults.sh | 2 +- src/lib.rs | 22 +--------------------- 3 files changed, 18 insertions(+), 22 deletions(-) create mode 100644 crates/utils/src/main.rs diff --git a/crates/utils/src/main.rs b/crates/utils/src/main.rs new file mode 100644 index 000000000..c2365f233 --- /dev/null +++ b/crates/utils/src/main.rs @@ -0,0 +1,16 @@ +use doku::json::{AutoComments, CommentsStyle, Formatting, ObjectsStyle}; +use lemmy_utils::settings::structs::Settings; +fn main() { + let fmt = Formatting { + auto_comments: AutoComments::none(), + comments_style: CommentsStyle { + separator: "#".to_owned(), + }, + objects_style: ObjectsStyle { + surround_keys_with_quotes: false, + use_comma_as_separator: false, + }, + ..Default::default() + }; + println!("{}", doku::to_json_fmt_val(&fmt, &Settings::default())); +} diff --git a/scripts/update_config_defaults.sh b/scripts/update_config_defaults.sh index 024b8ca11..0984c247c 100755 --- a/scripts/update_config_defaults.sh +++ b/scripts/update_config_defaults.sh @@ -3,4 +3,4 @@ set -e dest=${1-config/defaults.hjson} -cargo run -- --print-config-docs > "$dest" +cargo run --manifest-path crates/utils/Cargo.toml > "$dest" diff --git a/src/lib.rs b/src/lib.rs index d919acc05..a704262e8 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -9,7 +9,6 @@ use crate::{code_migrations::run_advanced_migrations, root_span_builder::Quieter use activitypub_federation::config::{FederationConfig, FederationMiddleware}; use actix_cors::Cors; use actix_web::{middleware, web::Data, App, HttpServer, Result}; -use doku::json::{AutoComments, CommentsStyle, Formatting, ObjectsStyle}; use lemmy_api_common::{ context::LemmyContext, lemmy_db_views::structs::SiteView, @@ -25,11 +24,7 @@ use lemmy_db_schema::{ utils::{build_db_pool, get_database_url, run_migrations}, }; use lemmy_routes::{feeds, images, nodeinfo, webfinger}; -use lemmy_utils::{ - error::LemmyError, - rate_limit::RateLimitCell, - settings::{structs::Settings, SETTINGS}, -}; +use lemmy_utils::{error::LemmyError, rate_limit::RateLimitCell, settings::SETTINGS}; use reqwest::Client; use reqwest_middleware::ClientBuilder; use reqwest_tracing::TracingMiddleware; @@ -47,21 +42,6 @@ pub(crate) const REQWEST_TIMEOUT: Duration = Duration::from_secs(10); /// Placing the main function in lib.rs allows other crates to import it and embed Lemmy pub async fn start_lemmy_server() -> Result<(), LemmyError> { let args: Vec = env::args().collect(); - if args.get(1) == Some(&"--print-config-docs".to_string()) { - let fmt = Formatting { - auto_comments: AutoComments::none(), - comments_style: CommentsStyle { - separator: "#".to_owned(), - }, - objects_style: ObjectsStyle { - surround_keys_with_quotes: false, - use_comma_as_separator: false, - }, - ..Default::default() - }; - println!("{}", doku::to_json_fmt_val(&fmt, &Settings::default())); - return Ok(()); - } let scheduled_tasks_enabled = args.get(1) != Some(&"--disable-scheduled-tasks".to_string()); From 73d2faa9f5773953eeda9402fc3934e0fdfa239c Mon Sep 17 00:00:00 2001 From: Felix Ableitner Date: Mon, 26 Jun 2023 12:46:44 +0200 Subject: [PATCH 23/45] Remove another unused dev dependency --- crates/db_schema/Cargo.toml | 1 - crates/utils/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index 6c89ab9fa..e99f3cd1c 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -51,4 +51,3 @@ tokio-postgres-rustls = { workspace = true } [dev-dependencies] serial_test = { workspace = true } - diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 1ec8d4ba2..47b55f735 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -49,7 +49,6 @@ enum-map = "2.5" [dev-dependencies] reqwest = { workspace = true } -tokio = { workspace = true, features = ["macros"] } [build-dependencies] rosetta-build = "0.1.2" From b2a9d4a335185b25779740a936a9ba8b20e07986 Mon Sep 17 00:00:00 2001 From: dullbananas Date: Mon, 26 Jun 2023 03:54:41 -0700 Subject: [PATCH 24/45] Use compression middleware (#3343) --- Cargo.lock | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ Cargo.toml | 2 +- src/lib.rs | 1 + 3 files changed, 83 insertions(+), 1 deletion(-) diff --git a/Cargo.lock b/Cargo.lock index 99d94948f..5a0a40d58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -122,10 +122,12 @@ dependencies = [ "ahash 0.8.3", "base64 0.21.2", "bitflags 1.3.2", + "brotli", "bytes", "bytestring", "derive_more", "encoding_rs", + "flate2", "futures-core", "h2", "http", @@ -143,6 +145,7 @@ dependencies = [ "tokio", "tokio-util 0.7.4", "tracing", + "zstd", ] [[package]] @@ -361,6 +364,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "alloc-no-stdlib" +version = "2.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc7bb162ec39d46ab1ca8c77bf72e890535becd1751bb45f64c597edb4c8c6b3" + +[[package]] +name = "alloc-stdlib" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94fb8275041c72129eb51b7d0322c29b8387a0386127718b096429201a5d6ece" +dependencies = [ + "alloc-no-stdlib", +] + [[package]] name = "android-tzdata" version = "0.1.1" @@ -714,6 +732,27 @@ dependencies = [ "cipher", ] +[[package]] +name = "brotli" +version = "3.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1a0b1dbcc8ae29329621f8d4f0d835787c1c38bb1401979b49d13b0b305ff68" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", + "brotli-decompressor", +] + +[[package]] +name = "brotli-decompressor" +version = "2.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b6561fd3f895a11e8f72af2cb7d22e08366bebc2b6b57f7744c4bda27034744" +dependencies = [ + "alloc-no-stdlib", + "alloc-stdlib", +] + [[package]] name = "bumpalo" version = "3.11.1" @@ -766,6 +805,9 @@ name = "cc" version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11" +dependencies = [ + "jobserver", +] [[package]] name = "cesu8" @@ -2470,6 +2512,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" +[[package]] +name = "jobserver" +version = "0.1.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "936cfd212a0155903bcbc060e316fb6cc7cbf2e1907329391ebadc1fe0ce77c2" +dependencies = [ + "libc", +] + [[package]] name = "js-sys" version = "0.3.60" @@ -6524,3 +6575,33 @@ name = "zeroize" version = "1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f" + +[[package]] +name = "zstd" +version = "0.12.3+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76eea132fb024e0e13fd9c2f5d5d595d8a967aa72382ac2f9d39fcc95afd0806" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "6.0.5+zstd.1.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d56d9e60b4b1758206c238a10165fbcae3ca37b01744e394c463463f6529d23b" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.8+zstd.1.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5556e6ee25d32df2586c098bbfa278803692a20d0ab9565e049480d52707ec8c" +dependencies = [ + "cc", + "libc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index c2cbc4879..00b400dd1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -65,7 +65,7 @@ diesel_migrations = "2.1.0" diesel-async = "0.3.1" serde = { version = "1.0.164", features = ["derive"] } serde_with = "1.14.0" -actix-web = { version = "4.3.1", default-features = false, features = ["macros", "rustls"] } +actix-web = { version = "4.3.1", default-features = false, features = ["macros", "rustls", "compress-brotli", "compress-gzip", "compress-zstd"] } tracing = "0.1.37" tracing-actix-web = { version = "0.6.2", default-features = false } tracing-error = "0.2.0" diff --git a/src/lib.rs b/src/lib.rs index a704262e8..cc77ca48f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -150,6 +150,7 @@ pub async fn start_lemmy_server() -> Result<(), LemmyError> { // This is the default log format save for the usage of %{r}a over %a to guarantee to record the client's (forwarded) IP and not the last peer address, since the latter is frequently just a reverse proxy "%{r}a '%r' %s %b '%{Referer}i' '%{User-Agent}i' %T", )) + .wrap(middleware::Compress::default()) .wrap(cors_config) .wrap(TracingLogger::::new()) .app_data(Data::new(context.clone())) From 62c8ac1db50f25c96fc1010b4f41d9eb450543f0 Mon Sep 17 00:00:00 2001 From: Otto Rottier Date: Mon, 26 Jun 2023 15:07:57 +0200 Subject: [PATCH 25/45] Site Metadata: resolve relative URLs for embedded images/videos (#3338) * Site Metadata: resolve relative URLs for embedded images/videos * api_common: relax version requirement of `webpage` dependency With this change we opt into next (non breaking) versions of webpage-rs * cargo +nightly fmt * Add tests for resolving absolute urls in SiteMetadata --- crates/api_common/Cargo.toml | 2 +- crates/api_common/src/request.rs | 59 +++++++++++++++++++++++++++++--- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/crates/api_common/Cargo.toml b/crates/api_common/Cargo.toml index 339d233a1..a9b2bf19b 100644 --- a/crates/api_common/Cargo.toml +++ b/crates/api_common/Cargo.toml @@ -33,7 +33,7 @@ reqwest-middleware = { workspace = true, optional = true } regex = { workspace = true } rosetta-i18n = { workspace = true, optional = true } percent-encoding = { workspace = true, optional = true } -webpage = { version = "1.6.0", default-features = false, features = ["serde"], optional = true } +webpage = { version = "1.6", default-features = false, features = ["serde"], optional = true } encoding = { version = "0.2.33", optional = true } anyhow = { workspace = true } futures = { workspace = true } diff --git a/crates/api_common/src/request.rs b/crates/api_common/src/request.rs index 3139193a6..9f7f9db59 100644 --- a/crates/api_common/src/request.rs +++ b/crates/api_common/src/request.rs @@ -27,12 +27,12 @@ pub async fn fetch_site_metadata( // https://github.com/LemmyNet/lemmy/issues/1964 let html_bytes = response.bytes().await.map_err(LemmyError::from)?.to_vec(); - let tags = html_to_site_metadata(&html_bytes)?; + let tags = html_to_site_metadata(&html_bytes, url)?; Ok(tags) } -fn html_to_site_metadata(html_bytes: &[u8]) -> Result { +fn html_to_site_metadata(html_bytes: &[u8], url: &Url) -> Result { let html = String::from_utf8_lossy(html_bytes); // Make sure the first line is doctype html @@ -81,12 +81,14 @@ fn html_to_site_metadata(html_bytes: &[u8]) -> Result .opengraph .images .first() - .and_then(|ogo| Url::parse(&ogo.url).ok()); + // join also works if the target URL is absolute + .and_then(|ogo| url.join(&ogo.url).ok()); let og_embed_url = page .opengraph .videos .first() - .and_then(|v| Url::parse(&v.url).ok()); + // join also works if the target URL is absolute + .and_then(|v| url.join(&v.url).ok()); Ok(SiteMetadata { title: og_title.or(page_title), @@ -266,7 +268,12 @@ pub fn build_user_agent(settings: &Settings) -> String { #[cfg(test)] mod tests { - use crate::request::{build_user_agent, fetch_site_metadata, SiteMetadata}; + use crate::request::{ + build_user_agent, + fetch_site_metadata, + html_to_site_metadata, + SiteMetadata, + }; use lemmy_utils::settings::SETTINGS; use url::Url; @@ -305,4 +312,46 @@ mod tests { // let res_other = fetch_pictshare("https://upload.wikimedia.org/wikipedia/en/2/27/The_Mandalorian_logo.jpgaoeu"); // assert!(res_other.is_err()); // } + + #[test] + fn test_resolve_image_url() { + // url that lists the opengraph fields + let url = Url::parse("https://example.com/one/two.html").unwrap(); + + // root relative url + let html_bytes = b""; + let metadata = html_to_site_metadata(html_bytes, &url).expect("Unable to parse metadata"); + assert_eq!( + metadata.image, + Some(Url::parse("https://example.com/image.jpg").unwrap().into()) + ); + + // base relative url + let html_bytes = b""; + let metadata = html_to_site_metadata(html_bytes, &url).expect("Unable to parse metadata"); + assert_eq!( + metadata.image, + Some( + Url::parse("https://example.com/one/image.jpg") + .unwrap() + .into() + ) + ); + + // absolute url + let html_bytes = b""; + let metadata = html_to_site_metadata(html_bytes, &url).expect("Unable to parse metadata"); + assert_eq!( + metadata.image, + Some(Url::parse("https://cdn.host.com/image.jpg").unwrap().into()) + ); + + // protocol relative url + let html_bytes = b""; + let metadata = html_to_site_metadata(html_bytes, &url).expect("Unable to parse metadata"); + assert_eq!( + metadata.image, + Some(Url::parse("https://example.com/image.jpg").unwrap().into()) + ); + } } From c9e9ff46faa40e2642343effb117693bfa525c5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Simon=20Lundstr=C3=B6m?= Date: Mon, 26 Jun 2023 15:11:16 +0200 Subject: [PATCH 26/45] Item URL should point to post URL (#3345) If the post is an URL post the item link should point to the URL of the link that is being posted. --- crates/routes/src/feeds.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/routes/src/feeds.rs b/crates/routes/src/feeds.rs index 3b4c2cd77..2d894c683 100644 --- a/crates/routes/src/feeds.rs +++ b/crates/routes/src/feeds.rs @@ -475,7 +475,6 @@ fn create_post_items( i.pub_date(dt.to_rfc2822()); let post_url = format!("{}/post/{}", protocol_and_hostname, p.post.id); - i.link(post_url.clone()); i.comments(post_url.clone()); let guid = GuidBuilder::default() .permalink(true) @@ -499,6 +498,9 @@ fn create_post_items( if let Some(url) = p.post.url { let link_html = format!("
{url}"); description.push_str(&link_html); + i.link(url.to_string()); + } else { + i.link(post_url.clone()); } if let Some(body) = p.post.body { From 8fbc630ce17dbeeb47933899fb84e8d3fb3ff1c1 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Mon, 26 Jun 2023 18:10:38 +0200 Subject: [PATCH 27/45] Remove networks from docker-compose.yml (#3356) --- docker/docker-compose.yml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 00059cd15..985eae5e6 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -6,20 +6,9 @@ x-logging: &default-logging max-size: "50m" max-file: 4 -networks: - # communication to web and clients - lemmyexternalproxy: - # communication between lemmy services - lemmyinternal: - driver: bridge - internal: true - services: proxy: image: nginx:1-alpine - networks: - - lemmyinternal - - lemmyexternalproxy ports: # actual and only port facing any connection from outside # Note, change the left number if port 1236 is already in use on your system @@ -45,9 +34,6 @@ services: # RUST_RELEASE_MODE: release # this hostname is used in nginx reverse proxy and also for lemmy ui to connect to the backend, do not change hostname: lemmy - networks: - - lemmyinternal - - lemmyexternalproxy restart: always environment: - RUST_LOG="warn,lemmy_server=debug,lemmy_api=debug,lemmy_api_common=debug,lemmy_api_crud=debug,lemmy_apub=debug,lemmy_db_schema=debug,lemmy_db_views=debug,lemmy_db_views_actor=debug,lemmy_db_views_moderator=debug,lemmy_routes=debug,lemmy_utils=debug,lemmy_websocket=debug" @@ -67,8 +53,6 @@ services: # build: # context: ../../lemmy-ui # dockerfile: dev.dockerfile - networks: - - lemmyinternal environment: # this needs to match the hostname defined in the lemmy service - LEMMY_UI_LEMMY_INTERNAL_HOST=lemmy:8536 @@ -88,8 +72,6 @@ services: hostname: pictrs # we can set options to pictrs like this, here we set max. image size and forced format for conversion # entrypoint: /sbin/tini -- /usr/local/bin/pict-rs -p /mnt -m 4 --image-format webp - networks: - - lemmyinternal environment: - PICTRS_OPENTELEMETRY_URL=http://otel:4137 - PICTRS__API_KEY=API_KEY @@ -126,10 +108,6 @@ services: "-c", "track_activity_query_size=1048576", ] - networks: - - lemmyinternal - # adding the external facing network to allow direct db access for devs - - lemmyexternalproxy ports: # use a different port so it doesnt conflict with potential postgres db running on the host - "5433:5432" From 50efb1d519c63a7007a07f11cc8a11487703c70d Mon Sep 17 00:00:00 2001 From: Nutomic Date: Mon, 26 Jun 2023 18:14:50 +0200 Subject: [PATCH 28/45] Fetch community outbox and moderators in parallel (#3360) This will speedup first time fetching of a remote community --- crates/apub/src/objects/community.rs | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/crates/apub/src/objects/community.rs b/crates/apub/src/objects/community.rs index 888a7f458..17476e9f8 100644 --- a/crates/apub/src/objects/community.rs +++ b/crates/apub/src/objects/community.rs @@ -140,19 +140,16 @@ impl Object for ApubCommunity { // Fetching mods and outbox is not necessary for Lemmy to work, so ignore errors. Besides, // we need to ignore these errors so that tests can work entirely offline. - group - .outbox - .dereference(&community, context) - .await - .map_err(|e| debug!("{}", e)) - .ok(); + let fetch_outbox = group.outbox.dereference(&community, context); if let Some(moderators) = group.attributed_to { - moderators - .dereference(&community, context) - .await - .map_err(|e| debug!("{}", e)) - .ok(); + let fetch_moderators = moderators.dereference(&community, context); + // Fetch mods and outbox in parallel + let res = tokio::join!(fetch_outbox, fetch_moderators); + res.0.map_err(|e| debug!("{}", e)).ok(); + res.1.map_err(|e| debug!("{}", e)).ok(); + } else { + fetch_outbox.await.map_err(|e| debug!("{}", e)).ok(); } Ok(community) From f5209fffc1de527db7ea007d463c158b36fda515 Mon Sep 17 00:00:00 2001 From: c-andy-candies <74613851+c-andy-candies@users.noreply.github.com> Date: Mon, 26 Jun 2023 21:03:35 +0200 Subject: [PATCH 29/45] Feature add three six and nine months options backend (#3226) Co-authored-by: Dessalines --- crates/db_schema/src/lib.rs | 3 +++ crates/db_schema/src/utils.rs | 5 ++++- crates/db_views/src/post_view.rs | 12 ++++++++++++ crates/db_views_actor/src/person_view.rs | 9 +++++++++ .../down.sql | 14 ++++++++++++++ .../up.sql | 4 ++++ 6 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 migrations/2023-06-20-191145_add_listingtype_sorttype_3_6_9_months_enums/down.sql create mode 100644 migrations/2023-06-20-191145_add_listingtype_sorttype_3_6_9_months_enums/up.sql diff --git a/crates/db_schema/src/lib.rs b/crates/db_schema/src/lib.rs index 04ec4e7da..acb069ca7 100644 --- a/crates/db_schema/src/lib.rs +++ b/crates/db_schema/src/lib.rs @@ -63,6 +63,9 @@ pub enum SortType { TopHour, TopSixHour, TopTwelveHour, + TopThreeMonths, + TopSixMonths, + TopNineMonths, } #[derive(EnumString, Display, Debug, Serialize, Deserialize, Clone, Copy)] diff --git a/crates/db_schema/src/utils.rs b/crates/db_schema/src/utils.rs index fdd445637..44230d10a 100644 --- a/crates/db_schema/src/utils.rs +++ b/crates/db_schema/src/utils.rs @@ -259,7 +259,10 @@ pub fn post_to_comment_sort_type(sort: SortType) -> CommentSortType { | SortType::TopAll | SortType::TopWeek | SortType::TopYear - | SortType::TopMonth => CommentSortType::Top, + | SortType::TopMonth + | SortType::TopThreeMonths + | SortType::TopSixMonths + | SortType::TopNineMonths => CommentSortType::Top, } } diff --git a/crates/db_views/src/post_view.rs b/crates/db_views/src/post_view.rs index 14cf7fe19..d1e974d8a 100644 --- a/crates/db_views/src/post_view.rs +++ b/crates/db_views/src/post_view.rs @@ -430,6 +430,18 @@ impl<'a> PostQuery<'a> { .filter(post_aggregates::published.gt(now - 12.hours())) .then_order_by(post_aggregates::score.desc()) .then_order_by(post_aggregates::published.desc()), + SortType::TopThreeMonths => query + .filter(post_aggregates::published.gt(now - 3.months())) + .then_order_by(post_aggregates::score.desc()) + .then_order_by(post_aggregates::published.desc()), + SortType::TopSixMonths => query + .filter(post_aggregates::published.gt(now - 6.months())) + .then_order_by(post_aggregates::score.desc()) + .then_order_by(post_aggregates::published.desc()), + SortType::TopNineMonths => query + .filter(post_aggregates::published.gt(now - 9.months())) + .then_order_by(post_aggregates::score.desc()) + .then_order_by(post_aggregates::published.desc()), }; let (limit, offset) = limit_and_offset(self.page, self.limit)?; diff --git a/crates/db_views_actor/src/person_view.rs b/crates/db_views_actor/src/person_view.rs index c7876cc1c..2a7a2ce79 100644 --- a/crates/db_views_actor/src/person_view.rs +++ b/crates/db_views_actor/src/person_view.rs @@ -122,6 +122,15 @@ impl<'a> PersonQuery<'a> { SortType::TopTwelveHour => query .filter(person::published.gt(now - 12.hours())) .order_by(person_aggregates::comment_score.desc()), + SortType::TopThreeMonths => query + .filter(person::published.gt(now - 3.months())) + .order_by(person_aggregates::comment_score.desc()), + SortType::TopSixMonths => query + .filter(person::published.gt(now - 6.months())) + .order_by(person_aggregates::comment_score.desc()), + SortType::TopNineMonths => query + .filter(person::published.gt(now - 9.months())) + .order_by(person_aggregates::comment_score.desc()), }; let (limit, offset) = limit_and_offset(self.page, self.limit)?; diff --git a/migrations/2023-06-20-191145_add_listingtype_sorttype_3_6_9_months_enums/down.sql b/migrations/2023-06-20-191145_add_listingtype_sorttype_3_6_9_months_enums/down.sql new file mode 100644 index 000000000..f8493fd26 --- /dev/null +++ b/migrations/2023-06-20-191145_add_listingtype_sorttype_3_6_9_months_enums/down.sql @@ -0,0 +1,14 @@ +-- update the default sort type +update local_user set default_sort_type = 'TopDay' where default_sort_type in ('TopThreeMonths', 'TopSixMonths', 'TopNineMonths'); + +-- rename the old enum +alter type sort_type_enum rename to sort_type_enum__; +-- create the new enum +CREATE TYPE sort_type_enum AS ENUM ('Active', 'Hot', 'New', 'Old', 'TopDay', 'TopWeek', 'TopMonth', 'TopYear', 'TopAll', 'MostComments', 'NewComments'); + +-- alter all you enum columns +alter table local_user + alter column default_sort_type type sort_type_enum using default_sort_type::text::sort_type_enum; + +-- drop the old enum +drop type sort_type_enum__; diff --git a/migrations/2023-06-20-191145_add_listingtype_sorttype_3_6_9_months_enums/up.sql b/migrations/2023-06-20-191145_add_listingtype_sorttype_3_6_9_months_enums/up.sql new file mode 100644 index 000000000..85bcfad7c --- /dev/null +++ b/migrations/2023-06-20-191145_add_listingtype_sorttype_3_6_9_months_enums/up.sql @@ -0,0 +1,4 @@ +-- Update the enums +ALTER TYPE sort_type_enum ADD VALUE 'TopThreeMonths'; +ALTER TYPE sort_type_enum ADD VALUE 'TopSixMonths'; +ALTER TYPE sort_type_enum ADD VALUE 'TopNineMonths'; From 211e76dc2781606331a2825b48a2b11fcd237d98 Mon Sep 17 00:00:00 2001 From: Sander Saarend Date: Tue, 27 Jun 2023 11:13:51 +0300 Subject: [PATCH 30/45] Batch hot rank updates (#3175) --- Cargo.lock | 1 + Cargo.toml | 2 +- .../down.sql | 2 + .../up.sql | 4 + src/scheduled_tasks.rs | 149 ++++++++++-------- 5 files changed, 92 insertions(+), 66 deletions(-) create mode 100644 migrations/2023-06-24-185942_aggegates_published_indexes/down.sql create mode 100644 migrations/2023-06-24-185942_aggegates_published_indexes/up.sql diff --git a/Cargo.lock b/Cargo.lock index 5a0a40d58..b01dd41c6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2786,6 +2786,7 @@ dependencies = [ "activitypub_federation", "actix-cors", "actix-web", + "chrono", "clokwerk", "console-subscriber", "diesel", diff --git a/Cargo.toml b/Cargo.toml index 00b400dd1..49d176b42 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -147,4 +147,4 @@ rustls = { workspace = true } futures-util = { workspace = true } tokio-postgres = { workspace = true } tokio-postgres-rustls = { workspace = true } - +chrono = { workspace = true } \ No newline at end of file diff --git a/migrations/2023-06-24-185942_aggegates_published_indexes/down.sql b/migrations/2023-06-24-185942_aggegates_published_indexes/down.sql new file mode 100644 index 000000000..fa7f7d48f --- /dev/null +++ b/migrations/2023-06-24-185942_aggegates_published_indexes/down.sql @@ -0,0 +1,2 @@ +drop index idx_comment_aggregates_published; +drop index idx_community_aggregates_published; diff --git a/migrations/2023-06-24-185942_aggegates_published_indexes/up.sql b/migrations/2023-06-24-185942_aggegates_published_indexes/up.sql new file mode 100644 index 000000000..42230af10 --- /dev/null +++ b/migrations/2023-06-24-185942_aggegates_published_indexes/up.sql @@ -0,0 +1,4 @@ +-- Add indexes on published column (needed for hot_rank updates) + +create index idx_community_aggregates_published on community_aggregates (published desc); +create index idx_comment_aggregates_published on comment_aggregates (published desc); \ No newline at end of file diff --git a/src/scheduled_tasks.rs b/src/scheduled_tasks.rs index 891dca365..c67dac0a4 100644 --- a/src/scheduled_tasks.rs +++ b/src/scheduled_tasks.rs @@ -1,28 +1,21 @@ +use chrono::NaiveDateTime; use clokwerk::{Scheduler, TimeUnits as CTimeUnits}; use diesel::{ dsl::{now, IntervalDsl}, + sql_types::{Integer, Timestamp}, Connection, ExpressionMethods, NullableExpressionMethods, QueryDsl, + QueryableByName, }; // Import week days and WeekDay use diesel::{sql_query, PgConnection, RunQueryDsl}; use lemmy_api_common::context::LemmyContext; use lemmy_db_schema::{ - schema::{ - activity, - comment, - comment_aggregates, - community_aggregates, - community_person_ban, - instance, - person, - post, - post_aggregates, - }, + schema::{activity, comment, community_person_ban, instance, person, post}, source::instance::{Instance, InstanceForm}, - utils::{functions::hot_rank, naive_now, DELETED_REPLACEMENT_TEXT}, + utils::{naive_now, DELETED_REPLACEMENT_TEXT}, }; use lemmy_routes::nodeinfo::NodeInfo; use lemmy_utils::{error::LemmyError, REQWEST_TIMEOUT}; @@ -49,9 +42,9 @@ pub fn setup( update_banned_when_expired(&mut conn); }); - // Update hot ranks every 5 minutes + // Update hot ranks every 15 minutes let url = db_url.clone(); - scheduler.every(CTimeUnits::minutes(5)).run(move || { + scheduler.every(CTimeUnits::minutes(15)).run(move || { let mut conn = PgConnection::establish(&url).expect("could not establish connection"); update_hot_ranks(&mut conn, true); }); @@ -100,66 +93,92 @@ fn startup_jobs(db_url: &str) { } /// Update the hot_rank columns for the aggregates tables +/// Runs in batches until all necessary rows are updated once fn update_hot_ranks(conn: &mut PgConnection, last_week_only: bool) { - let mut post_update = diesel::update(post_aggregates::table).into_boxed(); - let mut comment_update = diesel::update(comment_aggregates::table).into_boxed(); - let mut community_update = diesel::update(community_aggregates::table).into_boxed(); - - // Only update for the last week of content - if last_week_only { + let process_start_time = if last_week_only { info!("Updating hot ranks for last week..."); - let last_week = now - diesel::dsl::IntervalDsl::weeks(1); - - post_update = post_update.filter(post_aggregates::published.gt(last_week)); - comment_update = comment_update.filter(comment_aggregates::published.gt(last_week)); - community_update = community_update.filter(community_aggregates::published.gt(last_week)); + naive_now() - chrono::Duration::days(7) } else { info!("Updating hot ranks for all history..."); - } + NaiveDateTime::from_timestamp_opt(0, 0).expect("0 timestamp creation") + }; - match post_update - .set(( - post_aggregates::hot_rank.eq(hot_rank(post_aggregates::score, post_aggregates::published)), - post_aggregates::hot_rank_active.eq(hot_rank( - post_aggregates::score, - post_aggregates::newest_comment_time_necro, - )), + process_hot_ranks_in_batches( + conn, + "post_aggregates", + "SET hot_rank = hot_rank(a.score, a.published), + hot_rank_active = hot_rank(a.score, a.newest_comment_time_necro)", + process_start_time, + ); + + process_hot_ranks_in_batches( + conn, + "comment_aggregates", + "SET hot_rank = hot_rank(a.score, a.published)", + process_start_time, + ); + + process_hot_ranks_in_batches( + conn, + "community_aggregates", + "SET hot_rank = hot_rank(a.subscribers, a.published)", + process_start_time, + ); + + info!("Finished hot ranks update!"); +} + +#[derive(QueryableByName)] +struct HotRanksUpdateResult { + #[diesel(sql_type = Timestamp)] + published: NaiveDateTime, +} + +/// Runs the hot rank update query in batches until all rows after `process_start_time` have been +/// processed. +/// In `set_clause`, "a" will refer to the current aggregates table. +/// Locked rows are skipped in order to prevent deadlocks (they will likely get updated on the next +/// run) +fn process_hot_ranks_in_batches( + conn: &mut PgConnection, + table_name: &str, + set_clause: &str, + process_start_time: NaiveDateTime, +) { + let update_batch_size = 1000; // Bigger batches than this tend to cause seq scans + let mut previous_batch_result = Some(process_start_time); + while let Some(previous_batch_last_published) = previous_batch_result { + // Raw `sql_query` is used as a performance optimization - Diesel does not support doing this + // in a single query (neither as a CTE, nor using a subquery) + let result = sql_query(format!( + r#"WITH batch AS (SELECT a.id + FROM {aggregates_table} a + WHERE a.published > $1 + ORDER BY a.published + LIMIT $2 + FOR UPDATE SKIP LOCKED) + UPDATE {aggregates_table} a {set_clause} + FROM batch WHERE a.id = batch.id RETURNING a.published; + "#, + aggregates_table = table_name, + set_clause = set_clause )) - .execute(conn) - { - Ok(_) => {} - Err(e) => { - error!("Failed to update post_aggregates hot_ranks: {}", e) - } - } + .bind::(previous_batch_last_published) + .bind::(update_batch_size) + .get_results::(conn); - match comment_update - .set(comment_aggregates::hot_rank.eq(hot_rank( - comment_aggregates::score, - comment_aggregates::published, - ))) - .execute(conn) - { - Ok(_) => {} - Err(e) => { - error!("Failed to update comment_aggregates hot_ranks: {}", e) - } - } - - match community_update - .set(community_aggregates::hot_rank.eq(hot_rank( - community_aggregates::subscribers, - community_aggregates::published, - ))) - .execute(conn) - { - Ok(_) => { - info!("Done."); - } - Err(e) => { - error!("Failed to update community_aggregates hot_ranks: {}", e) + match result { + Ok(updated_rows) => previous_batch_result = updated_rows.last().map(|row| row.published), + Err(e) => { + error!("Failed to update {} hot_ranks: {}", table_name, e); + break; + } } } + info!( + "Finished process_hot_ranks_in_batches execution for {}", + table_name + ); } /// Clear old activities (this table gets very large) From 98482b1564ad7378f7ac439b5e5ab7247dcefc38 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Tue, 27 Jun 2023 04:28:56 -0400 Subject: [PATCH 31/45] Fixing the release script. (#3295) * Fixing the release script. * Updating the submodules. --- scripts/release.sh | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/scripts/release.sh b/scripts/release.sh index 4b9fdd8e1..2df168c43 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -10,15 +10,15 @@ third_semver=$(echo $new_tag | cut -d "." -f 3) # IE, when the third semver is a number, not '2-rc' if [ ! -z "${third_semver##*[!0-9]*}" ]; then pushd ../docker - sed -i "s/dessalines\/lemmy:.*/dessalines\/lemmy:$new_tag/" ../docker-compose.yml - sed -i "s/dessalines\/lemmy-ui:.*/dessalines\/lemmy-ui:$new_tag/" ../docker-compose.yml - sed -i "s/dessalines\/lemmy-ui:.*/dessalines\/lemmy-ui:$new_tag/" ../federation/docker-compose.yml - git add ../docker-compose.yml - git add ../federation/docker-compose.yml + sed -i "s/dessalines\/lemmy:.*/dessalines\/lemmy:$new_tag/" docker-compose.yml + sed -i "s/dessalines\/lemmy-ui:.*/dessalines\/lemmy-ui:$new_tag/" docker-compose.yml + sed -i "s/dessalines\/lemmy-ui:.*/dessalines\/lemmy-ui:$new_tag/" federation/docker-compose.yml + git add docker-compose.yml + git add federation/docker-compose.yml popd # Setting the version for Ansible - pushd ../../../lemmy-ansible + pushd ../../lemmy-ansible echo $new_tag > "VERSION" git add "VERSION" git commit -m"Updating VERSION" @@ -38,6 +38,10 @@ cargo check git add Cargo.lock popd +# Update the submodules +git submodule update --remote +git add crates/utils/translations + # The commit git commit -m"Version $new_tag" git tag $new_tag From 76a451377432a6a61889e06e7d7dbc146e7b797b Mon Sep 17 00:00:00 2001 From: Sander Saarend Date: Tue, 27 Jun 2023 12:20:53 +0300 Subject: [PATCH 32/45] Limit password resets (#3344) --- crates/api/src/local_user/reset_password.rs | 11 ++++++++++ .../src/impls/password_reset_request.rs | 20 ++++++++++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/crates/api/src/local_user/reset_password.rs b/crates/api/src/local_user/reset_password.rs index 3c07b2e4d..b5325608d 100644 --- a/crates/api/src/local_user/reset_password.rs +++ b/crates/api/src/local_user/reset_password.rs @@ -5,6 +5,7 @@ use lemmy_api_common::{ person::{PasswordReset, PasswordResetResponse}, utils::send_password_reset_email, }; +use lemmy_db_schema::source::password_reset_request::PasswordResetRequest; use lemmy_db_views::structs::LocalUserView; use lemmy_utils::error::LemmyError; @@ -25,6 +26,16 @@ impl Perform for PasswordReset { .await .map_err(|e| LemmyError::from_error_message(e, "couldnt_find_that_username_or_email"))?; + // Check for too many attempts (to limit potential abuse) + let recent_resets_count = PasswordResetRequest::get_recent_password_resets_count( + context.pool(), + local_user_view.local_user.id, + ) + .await?; + if recent_resets_count >= 3 { + return Err(LemmyError::from_message("password_reset_limit_reached")); + } + // Email the pure token to the user. send_password_reset_email(&local_user_view, context.pool(), context.settings()).await?; Ok(PasswordResetResponse {}) diff --git a/crates/db_schema/src/impls/password_reset_request.rs b/crates/db_schema/src/impls/password_reset_request.rs index ebe8a00ec..85ad4cf01 100644 --- a/crates/db_schema/src/impls/password_reset_request.rs +++ b/crates/db_schema/src/impls/password_reset_request.rs @@ -1,6 +1,11 @@ use crate::{ newtypes::LocalUserId, - schema::password_reset_request::dsl::{password_reset_request, published, token_encrypted}, + schema::password_reset_request::dsl::{ + local_user_id, + password_reset_request, + published, + token_encrypted, + }, source::password_reset_request::{PasswordResetRequest, PasswordResetRequestForm}, traits::Crud, utils::{get_conn, DbPool}, @@ -74,6 +79,19 @@ impl PasswordResetRequest { .first::(conn) .await } + + pub async fn get_recent_password_resets_count( + pool: &DbPool, + user_id: LocalUserId, + ) -> Result { + let conn = &mut get_conn(pool).await?; + password_reset_request + .filter(local_user_id.eq(user_id)) + .filter(published.gt(now - 1.days())) + .count() + .get_result(conn) + .await + } } fn bytes_to_hex(bytes: Vec) -> String { From 2aef6a5a338c9a6b5764fbc1ae42fa3edf096deb Mon Sep 17 00:00:00 2001 From: TKilFree Date: Tue, 27 Jun 2023 11:38:53 +0100 Subject: [PATCH 33/45] feat: re-added captcha checks (#3289) --- Cargo.lock | 5 + crates/api/Cargo.toml | 1 + crates/api/src/lib.rs | 16 +++ crates/api/src/local_user/get_captcha.rs | 50 ++++++++ crates/api/src/local_user/mod.rs | 1 + crates/api_crud/Cargo.toml | 2 + crates/api_crud/src/user/create.rs | 20 +++ crates/db_schema/Cargo.toml | 3 +- crates/db_schema/src/diesel_ltree.patch | 4 +- crates/db_schema/src/impls/captcha_answer.rs | 118 ++++++++++++++++++ crates/db_schema/src/impls/mod.rs | 1 + crates/db_schema/src/schema.rs | 10 ++ crates/db_schema/src/source/captcha_answer.rs | 33 +++++ crates/db_schema/src/source/mod.rs | 1 + .../2023-06-21-153242_add_captcha/down.sql | 1 + .../2023-06-21-153242_add_captcha/up.sql | 6 + src/api_routes_http.rs | 7 ++ src/scheduled_tasks.rs | 24 +++- 18 files changed, 299 insertions(+), 4 deletions(-) create mode 100644 crates/api/src/local_user/get_captcha.rs create mode 100644 crates/db_schema/src/impls/captcha_answer.rs create mode 100644 crates/db_schema/src/source/captcha_answer.rs create mode 100644 migrations/2023-06-21-153242_add_captcha/down.sql create mode 100644 migrations/2023-06-21-153242_add_captcha/up.sql diff --git a/Cargo.lock b/Cargo.lock index b01dd41c6..39ef0d807 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1370,6 +1370,7 @@ dependencies = [ "itoa", "pq-sys", "serde_json", + "uuid", ] [[package]] @@ -2577,6 +2578,7 @@ dependencies = [ "base64 0.13.1", "bcrypt", "captcha", + "chrono", "lemmy_api_common", "lemmy_db_schema", "lemmy_db_views", @@ -2627,6 +2629,7 @@ dependencies = [ "actix-web", "async-trait", "bcrypt", + "chrono", "lemmy_api_common", "lemmy_db_schema", "lemmy_db_views", @@ -2635,6 +2638,7 @@ dependencies = [ "serde", "tracing", "url", + "uuid", "webmention", ] @@ -2710,6 +2714,7 @@ dependencies = [ "ts-rs", "typed-builder", "url", + "uuid", ] [[package]] diff --git a/crates/api/Cargo.toml b/crates/api/Cargo.toml index 2488f2c2c..ca792809b 100644 --- a/crates/api/Cargo.toml +++ b/crates/api/Cargo.toml @@ -29,6 +29,7 @@ async-trait = { workspace = true } captcha = { workspace = true } anyhow = { workspace = true } tracing = { workspace = true } +chrono = { workspace = true } [dev-dependencies] serial_test = { workspace = true } diff --git a/crates/api/src/lib.rs b/crates/api/src/lib.rs index 9ff1677d0..615a8a314 100644 --- a/crates/api/src/lib.rs +++ b/crates/api/src/lib.rs @@ -1,4 +1,5 @@ use actix_web::web::Data; +use captcha::Captcha; use lemmy_api_common::{context::LemmyContext, utils::local_site_to_slur_regex}; use lemmy_db_schema::source::local_site::LocalSite; use lemmy_utils::{error::LemmyError, utils::slurs::check_slurs}; @@ -20,6 +21,21 @@ pub trait Perform { async fn perform(&self, context: &Data) -> Result; } +/// Converts the captcha to a base64 encoded wav audio file +pub(crate) fn captcha_as_wav_base64(captcha: &Captcha) -> String { + let letters = captcha.as_wav(); + + let mut concat_letters: Vec = Vec::new(); + + for letter in letters { + let bytes = letter.unwrap_or_default(); + concat_letters.extend(bytes); + } + + // Convert to base64 + base64::encode(concat_letters) +} + /// Check size of report and remove whitespace pub(crate) fn check_report_reason(reason: &str, local_site: &LocalSite) -> Result<(), LemmyError> { let slur_regex = &local_site_to_slur_regex(local_site); diff --git a/crates/api/src/local_user/get_captcha.rs b/crates/api/src/local_user/get_captcha.rs new file mode 100644 index 000000000..133044248 --- /dev/null +++ b/crates/api/src/local_user/get_captcha.rs @@ -0,0 +1,50 @@ +use crate::{captcha_as_wav_base64, Perform}; +use actix_web::web::Data; +use captcha::{gen, Difficulty}; +use lemmy_api_common::{ + context::LemmyContext, + person::{CaptchaResponse, GetCaptcha, GetCaptchaResponse}, +}; +use lemmy_db_schema::source::{ + captcha_answer::{CaptchaAnswer, CaptchaAnswerForm}, + local_site::LocalSite, +}; +use lemmy_utils::error::LemmyError; + +#[async_trait::async_trait(?Send)] +impl Perform for GetCaptcha { + type Response = GetCaptchaResponse; + + #[tracing::instrument(skip(context))] + async fn perform(&self, context: &Data) -> Result { + let local_site = LocalSite::read(context.pool()).await?; + + if !local_site.captcha_enabled { + return Ok(GetCaptchaResponse { ok: None }); + } + + let captcha = gen(match local_site.captcha_difficulty.as_str() { + "easy" => Difficulty::Easy, + "hard" => Difficulty::Hard, + _ => Difficulty::Medium, + }); + + let answer = captcha.chars_as_string(); + + let png = captcha.as_base64().expect("failed to generate captcha"); + + let wav = captcha_as_wav_base64(&captcha); + + let captcha_form: CaptchaAnswerForm = CaptchaAnswerForm { answer }; + // Stores the captcha item in the db + let captcha = CaptchaAnswer::insert(context.pool(), &captcha_form).await?; + + Ok(GetCaptchaResponse { + ok: Some(CaptchaResponse { + png, + wav, + uuid: captcha.uuid.to_string(), + }), + }) + } +} diff --git a/crates/api/src/local_user/mod.rs b/crates/api/src/local_user/mod.rs index 9244f825d..3a92beda5 100644 --- a/crates/api/src/local_user/mod.rs +++ b/crates/api/src/local_user/mod.rs @@ -3,6 +3,7 @@ mod ban_person; mod block; mod change_password; mod change_password_after_reset; +mod get_captcha; mod list_banned; mod login; mod notifications; diff --git a/crates/api_crud/Cargo.toml b/crates/api_crud/Cargo.toml index 1fb1e5a66..abe747b15 100644 --- a/crates/api_crud/Cargo.toml +++ b/crates/api_crud/Cargo.toml @@ -22,3 +22,5 @@ tracing = { workspace = true } url = { workspace = true } async-trait = { workspace = true } webmention = "0.4.0" +chrono = { worspace = true } +uuid = { workspace = true } \ No newline at end of file diff --git a/crates/api_crud/src/user/create.rs b/crates/api_crud/src/user/create.rs index f5a26f756..302e2f98e 100644 --- a/crates/api_crud/src/user/create.rs +++ b/crates/api_crud/src/user/create.rs @@ -19,6 +19,7 @@ use lemmy_api_common::{ use lemmy_db_schema::{ aggregates::structs::PersonAggregates, source::{ + captcha_answer::{CaptchaAnswer, CheckCaptchaAnswer}, local_user::{LocalUser, LocalUserInsertForm}, person::{Person, PersonInsertForm}, registration_application::{RegistrationApplication, RegistrationApplicationInsertForm}, @@ -71,6 +72,25 @@ impl PerformCrud for Register { return Err(LemmyError::from_message("passwords_dont_match")); } + if local_site.site_setup && local_site.captcha_enabled { + if let Some(captcha_uuid) = &data.captcha_uuid { + let uuid = uuid::Uuid::parse_str(captcha_uuid)?; + let check = CaptchaAnswer::check_captcha( + context.pool(), + CheckCaptchaAnswer { + uuid, + answer: data.captcha_answer.clone().unwrap_or_default(), + }, + ) + .await?; + if !check { + return Err(LemmyError::from_message("captcha_incorrect")); + } + } else { + return Err(LemmyError::from_message("captcha_incorrect")); + } + } + let slur_regex = local_site_to_slur_regex(&local_site); check_slurs(&data.username, &slur_regex)?; check_slurs_opt(&data.answer, &slur_regex)?; diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index e99f3cd1c..69affde88 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -29,7 +29,7 @@ serde_json = { workspace = true, optional = true } activitypub_federation = { workspace = true, optional = true } lemmy_utils = { workspace = true, optional = true } bcrypt = { workspace = true, optional = true } -diesel = { workspace = true, features = ["postgres","chrono", "serde_json"], optional = true } +diesel = { workspace = true, features = ["postgres","chrono", "serde_json", "uuid"], optional = true } diesel-derive-newtype = { workspace = true, optional = true } diesel-derive-enum = { workspace = true, optional = true } diesel_migrations = { workspace = true, optional = true } @@ -48,6 +48,7 @@ rustls = { workspace = true } futures-util = { workspace = true } tokio-postgres = { workspace = true } tokio-postgres-rustls = { workspace = true } +uuid = { features = ["v4"] } [dev-dependencies] serial_test = { workspace = true } diff --git a/crates/db_schema/src/diesel_ltree.patch b/crates/db_schema/src/diesel_ltree.patch index d7d49f03e..ecbeb2193 100644 --- a/crates/db_schema/src/diesel_ltree.patch +++ b/crates/db_schema/src/diesel_ltree.patch @@ -19,8 +19,8 @@ index 255c6422..f2ccf5e2 100644 #[derive(diesel::sql_types::SqlType)] #[diesel(postgres_type(name = "sort_type_enum"))] -@@ -67,13 +63,13 @@ diesel::table! { - when_ -> Timestamp, +@@ -76,13 +76,13 @@ diesel::table! { + published -> Timestamp, } } diff --git a/crates/db_schema/src/impls/captcha_answer.rs b/crates/db_schema/src/impls/captcha_answer.rs new file mode 100644 index 000000000..de5fac65e --- /dev/null +++ b/crates/db_schema/src/impls/captcha_answer.rs @@ -0,0 +1,118 @@ +use crate::{ + schema::captcha_answer::dsl::{answer, captcha_answer, uuid}, + source::captcha_answer::{CaptchaAnswer, CaptchaAnswerForm, CheckCaptchaAnswer}, + utils::{functions::lower, get_conn, DbPool}, +}; +use diesel::{ + delete, + dsl::exists, + insert_into, + result::Error, + select, + ExpressionMethods, + QueryDsl, +}; +use diesel_async::RunQueryDsl; + +impl CaptchaAnswer { + pub async fn insert(pool: &DbPool, captcha: &CaptchaAnswerForm) -> Result { + let conn = &mut get_conn(pool).await?; + + insert_into(captcha_answer) + .values(captcha) + .get_result::(conn) + .await + } + + pub async fn check_captcha(pool: &DbPool, to_check: CheckCaptchaAnswer) -> Result { + let conn = &mut get_conn(pool).await?; + + // fetch requested captcha + let captcha_exists = select(exists( + captcha_answer + .filter((uuid).eq(to_check.uuid)) + .filter(lower(answer).eq(to_check.answer.to_lowercase().clone())), + )) + .get_result::(conn) + .await?; + + // delete checked captcha + delete(captcha_answer.filter(uuid.eq(to_check.uuid))) + .execute(conn) + .await?; + + Ok(captcha_exists) + } +} + +#[cfg(test)] +mod tests { + use crate::{ + source::captcha_answer::{CaptchaAnswer, CaptchaAnswerForm, CheckCaptchaAnswer}, + utils::build_db_pool_for_tests, + }; + use serial_test::serial; + + #[tokio::test] + #[serial] + async fn test_captcha_happy_path() { + let pool = &build_db_pool_for_tests().await; + + let inserted = CaptchaAnswer::insert( + pool, + &CaptchaAnswerForm { + answer: "XYZ".to_string(), + }, + ) + .await + .expect("should not fail to insert captcha"); + + let result = CaptchaAnswer::check_captcha( + pool, + CheckCaptchaAnswer { + uuid: inserted.uuid, + answer: "xyz".to_string(), + }, + ) + .await; + + assert!(result.is_ok()); + assert!(result.unwrap()); + } + + #[tokio::test] + #[serial] + async fn test_captcha_repeat_answer_fails() { + let pool = &build_db_pool_for_tests().await; + + let inserted = CaptchaAnswer::insert( + pool, + &CaptchaAnswerForm { + answer: "XYZ".to_string(), + }, + ) + .await + .expect("should not fail to insert captcha"); + + let _result = CaptchaAnswer::check_captcha( + pool, + CheckCaptchaAnswer { + uuid: inserted.uuid, + answer: "xyz".to_string(), + }, + ) + .await; + + let result_repeat = CaptchaAnswer::check_captcha( + pool, + CheckCaptchaAnswer { + uuid: inserted.uuid, + answer: "xyz".to_string(), + }, + ) + .await; + + assert!(result_repeat.is_ok()); + assert!(!result_repeat.unwrap()); + } +} diff --git a/crates/db_schema/src/impls/mod.rs b/crates/db_schema/src/impls/mod.rs index 915d1c8e2..f13004d01 100644 --- a/crates/db_schema/src/impls/mod.rs +++ b/crates/db_schema/src/impls/mod.rs @@ -1,5 +1,6 @@ pub mod activity; pub mod actor_language; +pub mod captcha_answer; pub mod comment; pub mod comment_reply; pub mod comment_report; diff --git a/crates/db_schema/src/schema.rs b/crates/db_schema/src/schema.rs index abd3ca22a..42946d699 100644 --- a/crates/db_schema/src/schema.rs +++ b/crates/db_schema/src/schema.rs @@ -64,6 +64,15 @@ diesel::table! { } } +diesel::table! { + captcha_answer (id) { + id -> Int4, + uuid -> Uuid, + answer -> Text, + published -> Timestamp, + } +} + diesel::table! { use diesel::sql_types::*; use diesel_ltree::sql_types::Ltree; @@ -914,6 +923,7 @@ diesel::allow_tables_to_appear_in_same_query!( admin_purge_community, admin_purge_person, admin_purge_post, + captcha_answer, comment, comment_aggregates, comment_like, diff --git a/crates/db_schema/src/source/captcha_answer.rs b/crates/db_schema/src/source/captcha_answer.rs new file mode 100644 index 000000000..e3e64c4eb --- /dev/null +++ b/crates/db_schema/src/source/captcha_answer.rs @@ -0,0 +1,33 @@ +#[cfg(feature = "full")] +use crate::schema::captcha_answer; +use serde::{Deserialize, Serialize}; +use serde_with::skip_serializing_none; +use uuid::Uuid; + +#[skip_serializing_none] +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[cfg_attr(feature = "full", derive(Queryable))] +#[cfg_attr(feature = "full", diesel(table_name = captcha_answer))] +pub struct CaptchaAnswer { + pub id: i32, + pub uuid: Uuid, + pub answer: String, + pub published: chrono::NaiveDateTime, +} + +#[skip_serializing_none] +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[cfg_attr(feature = "full", derive(Queryable))] +#[cfg_attr(feature = "full", diesel(table_name = captcha_answer))] +pub struct CheckCaptchaAnswer { + pub uuid: Uuid, + pub answer: String, +} + +#[skip_serializing_none] +#[derive(Clone, PartialEq, Eq, Debug, Serialize, Deserialize)] +#[cfg_attr(feature = "full", derive(Insertable, AsChangeset))] +#[cfg_attr(feature = "full", diesel(table_name = captcha_answer))] +pub struct CaptchaAnswerForm { + pub answer: String, +} diff --git a/crates/db_schema/src/source/mod.rs b/crates/db_schema/src/source/mod.rs index 9aab4b90b..926e23e73 100644 --- a/crates/db_schema/src/source/mod.rs +++ b/crates/db_schema/src/source/mod.rs @@ -1,6 +1,7 @@ #[cfg(feature = "full")] pub mod activity; pub mod actor_language; +pub mod captcha_answer; pub mod comment; pub mod comment_reply; pub mod comment_report; diff --git a/migrations/2023-06-21-153242_add_captcha/down.sql b/migrations/2023-06-21-153242_add_captcha/down.sql new file mode 100644 index 000000000..4e5b83042 --- /dev/null +++ b/migrations/2023-06-21-153242_add_captcha/down.sql @@ -0,0 +1 @@ +drop table captcha_answer; \ No newline at end of file diff --git a/migrations/2023-06-21-153242_add_captcha/up.sql b/migrations/2023-06-21-153242_add_captcha/up.sql new file mode 100644 index 000000000..5c566bc92 --- /dev/null +++ b/migrations/2023-06-21-153242_add_captcha/up.sql @@ -0,0 +1,6 @@ +create table captcha_answer ( + id serial primary key, + uuid uuid not null unique default gen_random_uuid(), + answer text not null, + published timestamp not null default now() +); diff --git a/src/api_routes_http.rs b/src/api_routes_http.rs index a2abfa690..375630a92 100644 --- a/src/api_routes_http.rs +++ b/src/api_routes_http.rs @@ -38,6 +38,7 @@ use lemmy_api_common::{ ChangePassword, DeleteAccount, GetBannedPersons, + GetCaptcha, GetPersonDetails, GetPersonMentions, GetReplies, @@ -272,6 +273,12 @@ pub fn config(cfg: &mut web::ServiceConfig, rate_limit: &RateLimitCell) { .wrap(rate_limit.register()) .route(web::post().to(route_post_crud::)), ) + .service( + // Handle captcha separately + web::resource("/user/get_captcha") + .wrap(rate_limit.post()) + .route(web::get().to(route_get::)), + ) // User actions .service( web::scope("/user") diff --git a/src/scheduled_tasks.rs b/src/scheduled_tasks.rs index c67dac0a4..4d3c936e8 100644 --- a/src/scheduled_tasks.rs +++ b/src/scheduled_tasks.rs @@ -13,7 +13,7 @@ use diesel::{ use diesel::{sql_query, PgConnection, RunQueryDsl}; use lemmy_api_common::context::LemmyContext; use lemmy_db_schema::{ - schema::{activity, comment, community_person_ban, instance, person, post}, + schema::{activity, captcha_answer, comment, community_person_ban, instance, person, post}, source::instance::{Instance, InstanceForm}, utils::{naive_now, DELETED_REPLACEMENT_TEXT}, }; @@ -49,6 +49,13 @@ pub fn setup( update_hot_ranks(&mut conn, true); }); + // Delete any captcha answers older than ten minutes, every ten minutes + let url = db_url.clone(); + scheduler.every(CTimeUnits::minutes(10)).run(move || { + let mut conn = PgConnection::establish(&url).expect("could not establish connection"); + delete_expired_captcha_answers(&mut conn); + }); + // Clear old activities every week let url = db_url.clone(); scheduler.every(CTimeUnits::weeks(1)).run(move || { @@ -181,6 +188,21 @@ fn process_hot_ranks_in_batches( ); } +fn delete_expired_captcha_answers(conn: &mut PgConnection) { + match diesel::delete( + captcha_answer::table.filter(captcha_answer::published.lt(now - IntervalDsl::minutes(10))), + ) + .execute(conn) + { + Ok(_) => { + info!("Done."); + } + Err(e) => { + error!("Failed to clear old captcha answers: {}", e) + } + } +} + /// Clear old activities (this table gets very large) fn clear_old_activities(conn: &mut PgConnection) { info!("Clearing old activities..."); From d1d90af0eb749ad79bc46ee81717e7166936ea89 Mon Sep 17 00:00:00 2001 From: Domenic Horner Date: Tue, 27 Jun 2023 18:45:26 +0800 Subject: [PATCH 34/45] add new flag to api (#3363) --- crates/api_common/src/community.rs | 1 + crates/api_crud/src/community/list.rs | 2 ++ crates/db_views_actor/src/community_view.rs | 5 +++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/crates/api_common/src/community.rs b/crates/api_common/src/community.rs index cb92d4c2e..ff6ed1271 100644 --- a/crates/api_common/src/community.rs +++ b/crates/api_common/src/community.rs @@ -76,6 +76,7 @@ pub struct CommunityResponse { pub struct ListCommunities { pub type_: Option, pub sort: Option, + pub show_nsfw: Option, pub page: Option, pub limit: Option, pub auth: Option>, diff --git a/crates/api_crud/src/community/list.rs b/crates/api_crud/src/community/list.rs index a58737d99..d37dd2dc2 100644 --- a/crates/api_crud/src/community/list.rs +++ b/crates/api_crud/src/community/list.rs @@ -27,12 +27,14 @@ impl PerformCrud for ListCommunities { let sort = data.sort; let listing_type = data.type_; + let show_nsfw = data.show_nsfw; let page = data.page; let limit = data.limit; let local_user = local_user_view.map(|l| l.local_user); let communities = CommunityQuery::builder() .pool(context.pool()) .listing_type(listing_type) + .show_nsfw(show_nsfw) .sort(sort) .local_user(local_user.as_ref()) .page(page) diff --git a/crates/db_views_actor/src/community_view.rs b/crates/db_views_actor/src/community_view.rs index c4f5920b7..826a0e884 100644 --- a/crates/db_views_actor/src/community_view.rs +++ b/crates/db_views_actor/src/community_view.rs @@ -126,6 +126,7 @@ pub struct CommunityQuery<'a> { local_user: Option<&'a LocalUser>, search_term: Option, is_mod_or_admin: Option, + show_nsfw: Option, page: Option, limit: Option, } @@ -203,8 +204,8 @@ impl<'a> CommunityQuery<'a> { query = query.filter(community_block::person_id.is_null()); query = query.filter(community::nsfw.eq(false).or(local_user::show_nsfw.eq(true))); } else { - // No person in request, only show nsfw communities if show_nsfw passed into request - if !self.local_user.map(|l| l.show_nsfw).unwrap_or(false) { + // No person in request, only show nsfw communities if show_nsfw is passed into request + if !self.show_nsfw.unwrap_or(false) { query = query.filter(community::nsfw.eq(false)); } } From e63aa80c3a3631f4713525d65c8223e1106355e8 Mon Sep 17 00:00:00 2001 From: Nina Blanson Date: Tue, 27 Jun 2023 06:03:30 -0500 Subject: [PATCH 35/45] Fixes #2900 - Checks slur regex to see if it is too permissive (#3146) * Fixes #2900 - Checks slur regex to see if it is too permissive along with small validation organization * Clean up variable names, add handler for valid empty string usecase * Update tests * Create validation function and add tests * Test clean up * Use payload value vs local site value to prevent stunlocking * Remove println added while testing * Fall back to local site regex if not provided from request * Attempt clean up of flaky comment_view tests * Pull in latest submodule * Move application, post check into functions, add more tests and improve test readability --------- Co-authored-by: Nutomic --- crates/api_common/src/utils.rs | 9 - crates/api_crud/src/site/create.rs | 513 ++++++++++++++++++++++++--- crates/api_crud/src/site/mod.rs | 88 ++++- crates/api_crud/src/site/update.rs | 474 ++++++++++++++++++++++--- crates/utils/src/utils/validation.rs | 208 ++++++++++- 5 files changed, 1177 insertions(+), 115 deletions(-) diff --git a/crates/api_common/src/utils.rs b/crates/api_common/src/utils.rs index 4781ce923..e3e761c90 100644 --- a/crates/api_common/src/utils.rs +++ b/crates/api_common/src/utils.rs @@ -299,15 +299,6 @@ pub fn password_length_check(pass: &str) -> Result<(), LemmyError> { } } -/// Checks the site description length -pub fn site_description_length_check(description: &str) -> Result<(), LemmyError> { - if description.len() > 150 { - Err(LemmyError::from_message("site_description_length_overflow")) - } else { - Ok(()) - } -} - /// Checks for a honeypot. If this field is filled, fail the rest of the function pub fn honeypot_check(honeypot: &Option) -> Result<(), LemmyError> { if honeypot.is_some() && honeypot != &Some(String::new()) { diff --git a/crates/api_crud/src/site/create.rs b/crates/api_crud/src/site/create.rs index e7486e63a..838d5bc40 100644 --- a/crates/api_crud/src/site/create.rs +++ b/crates/api_crud/src/site/create.rs @@ -1,4 +1,7 @@ -use crate::{site::check_application_question, PerformCrud}; +use crate::{ + site::{application_question_check, site_default_post_listing_type_check}, + PerformCrud, +}; use activitypub_federation::http_signatures::generate_actor_keypair; use actix_web::web::Data; use lemmy_api_common::{ @@ -8,9 +11,7 @@ use lemmy_api_common::{ generate_site_inbox_url, is_admin, local_site_rate_limit_to_rate_limit_config, - local_site_to_slur_regex, local_user_view_from_jwt, - site_description_length_check, }, }; use lemmy_db_schema::{ @@ -26,10 +27,16 @@ use lemmy_db_schema::{ }; use lemmy_db_views::structs::SiteView; use lemmy_utils::{ - error::LemmyError, + error::{LemmyError, LemmyResult}, utils::{ slurs::{check_slurs, check_slurs_opt}, - validation::{check_site_visibility_valid, is_valid_body_field}, + validation::{ + build_and_check_regex, + check_site_visibility_valid, + is_valid_body_field, + site_description_length_check, + site_name_length_check, + }, }, }; use url::Url; @@ -41,57 +48,23 @@ impl PerformCrud for CreateSite { #[tracing::instrument(skip(context))] async fn perform(&self, context: &Data) -> Result { let data: &CreateSite = self; - + let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; let local_site = LocalSite::read(context.pool()).await?; - if local_site.site_setup { - return Err(LemmyError::from_message("site_already_exists")); - }; - - let local_user_view = local_user_view_from_jwt(&data.auth, context).await?; - - // Make sure user is an admin + // Make sure user is an admin; other types of users should not create site data... is_admin(&local_user_view)?; - check_site_visibility_valid( - local_site.private_instance, - local_site.federation_enabled, - &data.private_instance, - &data.federation_enabled, - )?; - - let sidebar = diesel_option_overwrite(&data.sidebar); - let description = diesel_option_overwrite(&data.description); - let icon = diesel_option_overwrite_to_url(&data.icon)?; - let banner = diesel_option_overwrite_to_url(&data.banner)?; - - let slur_regex = local_site_to_slur_regex(&local_site); - check_slurs(&data.name, &slur_regex)?; - check_slurs_opt(&data.description, &slur_regex)?; - - if let Some(Some(desc)) = &description { - site_description_length_check(desc)?; - } - - is_valid_body_field(&data.sidebar, false)?; - - let application_question = diesel_option_overwrite(&data.application_question); - check_application_question( - &application_question, - data - .registration_mode - .unwrap_or(local_site.registration_mode), - )?; + validate_create_payload(&local_site, data)?; let actor_id: DbUrl = Url::parse(&context.settings().get_protocol_and_hostname())?.into(); let inbox_url = Some(generate_site_inbox_url(&actor_id)?); let keypair = generate_actor_keypair()?; let site_form = SiteUpdateForm::builder() .name(Some(data.name.clone())) - .sidebar(sidebar) - .description(description) - .icon(icon) - .banner(banner) + .sidebar(diesel_option_overwrite(&data.sidebar)) + .description(diesel_option_overwrite(&data.description)) + .icon(diesel_option_overwrite_to_url(&data.icon)?) + .banner(diesel_option_overwrite_to_url(&data.banner)?) .actor_id(Some(actor_id)) .last_refreshed_at(Some(naive_now())) .inbox_url(inbox_url) @@ -111,7 +84,7 @@ impl PerformCrud for CreateSite { .enable_nsfw(data.enable_nsfw) .community_creation_admin_only(data.community_creation_admin_only) .require_email_verification(data.require_email_verification) - .application_question(application_question) + .application_question(diesel_option_overwrite(&data.application_question)) .private_instance(data.private_instance) .default_theme(data.default_theme.clone()) .default_post_listing_type(data.default_post_listing_type) @@ -163,3 +136,449 @@ impl PerformCrud for CreateSite { }) } } + +fn validate_create_payload(local_site: &LocalSite, create_site: &CreateSite) -> LemmyResult<()> { + // Make sure the site hasn't already been set up... + if local_site.site_setup { + return Err(LemmyError::from_message("site_already_exists")); + }; + + // Check that the slur regex compiles, and returns the regex if valid... + // Prioritize using new slur regex from the request; if not provided, use the existing regex. + let slur_regex = build_and_check_regex( + &create_site + .slur_filter_regex + .as_deref() + .or(local_site.slur_filter_regex.as_deref()), + )?; + + site_name_length_check(&create_site.name)?; + check_slurs(&create_site.name, &slur_regex)?; + + if let Some(desc) = &create_site.description { + site_description_length_check(desc)?; + check_slurs_opt(&create_site.description, &slur_regex)?; + } + + site_default_post_listing_type_check(&create_site.default_post_listing_type)?; + + check_site_visibility_valid( + local_site.private_instance, + local_site.federation_enabled, + &create_site.private_instance, + &create_site.federation_enabled, + )?; + + // Ensure that the sidebar has fewer than the max num characters... + is_valid_body_field(&create_site.sidebar, false)?; + + application_question_check( + &local_site.application_question, + &create_site.application_question, + create_site + .registration_mode + .unwrap_or(local_site.registration_mode), + ) +} + +#[cfg(test)] +mod tests { + use crate::site::create::validate_create_payload; + use lemmy_api_common::site::CreateSite; + use lemmy_db_schema::{source::local_site::LocalSite, ListingType, RegistrationMode}; + + #[test] + fn test_validate_invalid_create_payload() { + let invalid_payloads = [ + ( + "CreateSite attempted on set up LocalSite", + "site_already_exists", + &generate_local_site( + true, + None::, + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_create_site( + String::from("site_name"), + None::, + None::, + None::, + None::, + None::, + None::, + None::, + None::, + ), + ), + ( + "CreateSite name matches LocalSite slur filter", + "slurs", + &generate_local_site( + false, + Some(String::from("(foo|bar)")), + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_create_site( + String::from("foo site_name"), + None::, + None::, + None::, + None::, + None::, + None::, + None::, + None::, + ), + ), + ( + "CreateSite name matches new slur filter", + "slurs", + &generate_local_site( + false, + Some(String::from("(foo|bar)")), + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_create_site( + String::from("zeta site_name"), + None::, + None::, + None::, + Some(String::from("(zeta|alpha)")), + None::, + None::, + None::, + None::, + ), + ), + ( + "CreateSite listing type is Subscribed, which is invalid", + "invalid_default_post_listing_type", + &generate_local_site( + false, + None::, + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_create_site( + String::from("site_name"), + None::, + None::, + Some(ListingType::Subscribed), + None::, + None::, + None::, + None::, + None::, + ), + ), + ( + "CreateSite is both private and federated", + "cant_enable_private_instance_and_federation_together", + &generate_local_site( + false, + None::, + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_create_site( + String::from("site_name"), + None::, + None::, + None::, + None::, + Some(true), + Some(true), + None::, + None::, + ), + ), + ( + "LocalSite is private, but CreateSite also makes it federated", + "cant_enable_private_instance_and_federation_together", + &generate_local_site( + false, + None::, + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_create_site( + String::from("site_name"), + None::, + None::, + None::, + None::, + None::, + Some(true), + None::, + None::, + ), + ), + ( + "CreateSite requires application, but neither it nor LocalSite has an application question", + "application_question_required", + &generate_local_site( + false, + None::, + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_create_site( + String::from("site_name"), + None::, + None::, + None::, + None::, + None::, + None::, + None::, + Some(RegistrationMode::RequireApplication), + ), + ), + ]; + + invalid_payloads.iter().enumerate().for_each( + |( + idx, + &(reason, expected_err, local_site, create_site), + )| { + match validate_create_payload( + local_site, + create_site, + ) { + Ok(_) => { + panic!( + "Got Ok, but validation should have failed with error: {} for reason: {}. invalid_payloads.nth({})", + expected_err, reason, idx + ) + } + Err(error) => { + assert!( + error.message.eq(&Some(String::from(expected_err))), + "Got Err {:?}, but should have failed with message: {} for reason: {}. invalid_payloads.nth({})", + error.message, + expected_err, + reason, + idx + ) + } + } + }, + ); + } + + #[test] + fn test_validate_valid_create_payload() { + let valid_payloads = [ + ( + "No changes between LocalSite and CreateSite", + &generate_local_site( + false, + None::, + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_create_site( + String::from("site_name"), + None::, + None::, + None::, + None::, + None::, + None::, + None::, + None::, + ), + ), + ( + "CreateSite allows clearing and changing values", + &generate_local_site( + false, + None::, + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_create_site( + String::from("site_name"), + Some(String::new()), + Some(String::new()), + Some(ListingType::All), + Some(String::new()), + Some(false), + Some(true), + Some(String::new()), + Some(RegistrationMode::Open), + ), + ), + ( + "CreateSite clears existing slur filter regex", + &generate_local_site( + false, + Some(String::from("(foo|bar)")), + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_create_site( + String::from("foo site_name"), + None::, + None::, + None::, + Some(String::new()), + None::, + None::, + None::, + None::, + ), + ), + ( + "LocalSite has application question and CreateSite now requires applications,", + &generate_local_site( + false, + None::, + true, + false, + Some(String::from("question")), + RegistrationMode::Open, + ), + &generate_create_site( + String::from("site_name"), + None::, + None::, + None::, + None::, + None::, + None::, + None::, + Some(RegistrationMode::RequireApplication), + ), + ), + ]; + + valid_payloads + .iter() + .enumerate() + .for_each(|(idx, &(reason, local_site, edit_site))| { + assert!( + validate_create_payload(local_site, edit_site).is_ok(), + "Got Err, but should have got Ok for reason: {}. valid_payloads.nth({})", + reason, + idx + ); + }) + } + + fn generate_local_site( + site_setup: bool, + site_slur_filter_regex: Option, + site_is_private: bool, + site_is_federated: bool, + site_application_question: Option, + site_registration_mode: RegistrationMode, + ) -> LocalSite { + LocalSite { + id: Default::default(), + site_id: Default::default(), + site_setup, + enable_downvotes: false, + enable_nsfw: false, + community_creation_admin_only: false, + require_email_verification: false, + application_question: site_application_question, + private_instance: site_is_private, + default_theme: String::new(), + default_post_listing_type: ListingType::All, + legal_information: None, + hide_modlog_mod_names: false, + application_email_admins: false, + slur_filter_regex: site_slur_filter_regex, + actor_name_max_length: 0, + federation_enabled: site_is_federated, + captcha_enabled: false, + captcha_difficulty: String::new(), + published: Default::default(), + updated: None, + registration_mode: site_registration_mode, + reports_email_admins: false, + } + } + + // Allow the test helper function to have too many arguments. + // It's either this or generate the entire struct each time for testing. + #[allow(clippy::too_many_arguments)] + fn generate_create_site( + site_name: String, + site_description: Option, + site_sidebar: Option, + site_listing_type: Option, + site_slur_filter_regex: Option, + site_is_private: Option, + site_is_federated: Option, + site_application_question: Option, + site_registration_mode: Option, + ) -> CreateSite { + CreateSite { + name: site_name, + sidebar: site_sidebar, + description: site_description, + icon: None, + banner: None, + enable_downvotes: None, + enable_nsfw: None, + community_creation_admin_only: None, + require_email_verification: None, + application_question: site_application_question, + private_instance: site_is_private, + default_theme: None, + default_post_listing_type: site_listing_type, + legal_information: None, + application_email_admins: None, + hide_modlog_mod_names: None, + discussion_languages: None, + slur_filter_regex: site_slur_filter_regex, + actor_name_max_length: None, + rate_limit_message: None, + rate_limit_message_per_second: None, + rate_limit_post: None, + rate_limit_post_per_second: None, + rate_limit_register: None, + rate_limit_register_per_second: None, + rate_limit_image: None, + rate_limit_image_per_second: None, + rate_limit_comment: None, + rate_limit_comment_per_second: None, + rate_limit_search: None, + rate_limit_search_per_second: None, + federation_enabled: site_is_federated, + federation_debug: None, + captcha_enabled: None, + captcha_difficulty: None, + allowed_instances: None, + blocked_instances: None, + taglines: None, + registration_mode: site_registration_mode, + auth: Default::default(), + } + } +} diff --git a/crates/api_crud/src/site/mod.rs b/crates/api_crud/src/site/mod.rs index d0c09b935..a98f2057c 100644 --- a/crates/api_crud/src/site/mod.rs +++ b/crates/api_crud/src/site/mod.rs @@ -1,19 +1,95 @@ -use lemmy_db_schema::RegistrationMode; -use lemmy_utils::error::LemmyError; +use lemmy_db_schema::{ListingType, RegistrationMode}; +use lemmy_utils::error::{LemmyError, LemmyResult}; mod create; mod read; mod update; -pub fn check_application_question( - application_question: &Option>, +/// Checks whether the default post listing type is valid for a site. +pub fn site_default_post_listing_type_check( + default_post_listing_type: &Option, +) -> LemmyResult<()> { + if let Some(listing_type) = default_post_listing_type { + // Only allow all or local as default listing types... + if listing_type != &ListingType::All && listing_type != &ListingType::Local { + Err(LemmyError::from_message( + "invalid_default_post_listing_type", + )) + } else { + Ok(()) + } + } else { + Ok(()) + } +} + +/// Checks whether the application question and registration mode align. +pub fn application_question_check( + current_application_question: &Option, + new_application_question: &Option, registration_mode: RegistrationMode, -) -> Result<(), LemmyError> { +) -> LemmyResult<()> { + let has_no_question: bool = + current_application_question.is_none() && new_application_question.is_none(); + let is_nullifying_question: bool = new_application_question == &Some(String::new()); + if registration_mode == RegistrationMode::RequireApplication - && application_question.as_ref().unwrap_or(&None).is_none() + && (has_no_question || is_nullifying_question) { Err(LemmyError::from_message("application_question_required")) } else { Ok(()) } } + +#[cfg(test)] +mod tests { + use crate::site::{application_question_check, site_default_post_listing_type_check}; + use lemmy_db_schema::{ListingType, RegistrationMode}; + + #[test] + fn test_site_default_post_listing_type_check() { + assert!(site_default_post_listing_type_check(&None::).is_ok()); + assert!(site_default_post_listing_type_check(&Some(ListingType::All)).is_ok()); + assert!(site_default_post_listing_type_check(&Some(ListingType::Local)).is_ok()); + assert!(site_default_post_listing_type_check(&Some(ListingType::Subscribed)).is_err()); + } + + #[test] + fn test_application_question_check() { + assert!( + application_question_check(&Some(String::from("q")), &Some(String::new()), RegistrationMode::RequireApplication).is_err(), + "Expected application to be invalid because an application is required, current question: {:?}, new question: {:?}", + "q", + String::new(), + ); + assert!( + application_question_check(&None, &None, RegistrationMode::RequireApplication).is_err(), + "Expected application to be invalid because an application is required, current question: {:?}, new question: {:?}", + None::, + None:: + ); + + assert!( + application_question_check(&None, &None, RegistrationMode::Open).is_ok(), + "Expected application to be valid because no application required, current question: {:?}, new question: {:?}, mode: {:?}", + None::, + None::, + RegistrationMode::Open + ); + assert!( + application_question_check(&None, &Some(String::from("q")), RegistrationMode::RequireApplication).is_ok(), + "Expected application to be valid because new application provided, current question: {:?}, new question: {:?}, mode: {:?}", + None::, + Some(String::from("q")), + RegistrationMode::RequireApplication + ); + assert!( + application_question_check(&Some(String::from("q")), &None, RegistrationMode::RequireApplication).is_ok(), + "Expected application to be valid because application existed, current question: {:?}, new question: {:?}, mode: {:?}", + Some(String::from("q")), + None::, + RegistrationMode::RequireApplication + ); + } +} diff --git a/crates/api_crud/src/site/update.rs b/crates/api_crud/src/site/update.rs index fa800a5a9..c9f97e8df 100644 --- a/crates/api_crud/src/site/update.rs +++ b/crates/api_crud/src/site/update.rs @@ -1,15 +1,12 @@ -use crate::{site::check_application_question, PerformCrud}; +use crate::{ + site::{application_question_check, site_default_post_listing_type_check}, + PerformCrud, +}; use actix_web::web::Data; use lemmy_api_common::{ context::LemmyContext, site::{EditSite, SiteResponse}, - utils::{ - is_admin, - local_site_rate_limit_to_rate_limit_config, - local_site_to_slur_regex, - local_user_view_from_jwt, - site_description_length_check, - }, + utils::{is_admin, local_site_rate_limit_to_rate_limit_config, local_user_view_from_jwt}, }; use lemmy_db_schema::{ source::{ @@ -24,15 +21,20 @@ use lemmy_db_schema::{ }, traits::Crud, utils::{diesel_option_overwrite, diesel_option_overwrite_to_url, naive_now}, - ListingType, RegistrationMode, }; use lemmy_db_views::structs::SiteView; use lemmy_utils::{ - error::LemmyError, + error::{LemmyError, LemmyResult}, utils::{ slurs::check_slurs_opt, - validation::{check_site_visibility_valid, is_valid_body_field}, + validation::{ + build_and_check_regex, + check_site_visibility_valid, + is_valid_body_field, + site_description_length_check, + site_name_length_check, + }, }, }; @@ -48,51 +50,17 @@ impl PerformCrud for EditSite { let local_site = site_view.local_site; let site = site_view.site; - // Make sure user is an admin + // Make sure user is an admin; other types of users should not update site data... is_admin(&local_user_view)?; - check_site_visibility_valid( - local_site.private_instance, - local_site.federation_enabled, - &data.private_instance, - &data.federation_enabled, - )?; - - let slur_regex = local_site_to_slur_regex(&local_site); - - check_slurs_opt(&data.name, &slur_regex)?; - check_slurs_opt(&data.description, &slur_regex)?; - - if let Some(desc) = &data.description { - site_description_length_check(desc)?; - } - - is_valid_body_field(&data.sidebar, false)?; - - let application_question = diesel_option_overwrite(&data.application_question); - check_application_question( - &application_question, - data - .registration_mode - .unwrap_or(local_site.registration_mode), - )?; - - if let Some(listing_type) = &data.default_post_listing_type { - // only allow all or local as default listing types - if listing_type != &ListingType::All && listing_type != &ListingType::Local { - return Err(LemmyError::from_message( - "invalid_default_post_listing_type", - )); - } - } + validate_update_payload(&local_site, data)?; if let Some(discussion_languages) = data.discussion_languages.clone() { SiteLanguage::update(context.pool(), discussion_languages.clone(), &site).await?; } - let name = data.name.clone(); let site_form = SiteUpdateForm::builder() - .name(name) + .name(data.name.clone()) .sidebar(diesel_option_overwrite(&data.sidebar)) .description(diesel_option_overwrite(&data.description)) .icon(diesel_option_overwrite_to_url(&data.icon)?) @@ -112,7 +80,7 @@ impl PerformCrud for EditSite { .enable_nsfw(data.enable_nsfw) .community_creation_admin_only(data.community_creation_admin_only) .require_email_verification(data.require_email_verification) - .application_question(application_question) + .application_question(diesel_option_overwrite(&data.application_question)) .private_instance(data.private_instance) .default_theme(data.default_theme.clone()) .default_post_listing_type(data.default_post_listing_type) @@ -204,3 +172,411 @@ impl PerformCrud for EditSite { Ok(res) } } + +fn validate_update_payload(local_site: &LocalSite, edit_site: &EditSite) -> LemmyResult<()> { + // Check that the slur regex compiles, and return the regex if valid... + // Prioritize using new slur regex from the request; if not provided, use the existing regex. + let slur_regex = build_and_check_regex( + &edit_site + .slur_filter_regex + .as_deref() + .or(local_site.slur_filter_regex.as_deref()), + )?; + + if let Some(name) = &edit_site.name { + // The name doesn't need to be updated, but if provided it cannot be blanked out... + site_name_length_check(name)?; + check_slurs_opt(&edit_site.name, &slur_regex)?; + } + + if let Some(desc) = &edit_site.description { + site_description_length_check(desc)?; + check_slurs_opt(&edit_site.description, &slur_regex)?; + } + + site_default_post_listing_type_check(&edit_site.default_post_listing_type)?; + + check_site_visibility_valid( + local_site.private_instance, + local_site.federation_enabled, + &edit_site.private_instance, + &edit_site.federation_enabled, + )?; + + // Ensure that the sidebar has fewer than the max num characters... + is_valid_body_field(&edit_site.sidebar, false)?; + + application_question_check( + &local_site.application_question, + &edit_site.application_question, + edit_site + .registration_mode + .unwrap_or(local_site.registration_mode), + ) +} + +#[cfg(test)] +mod tests { + use crate::site::update::validate_update_payload; + use lemmy_api_common::site::EditSite; + use lemmy_db_schema::{source::local_site::LocalSite, ListingType, RegistrationMode}; + + #[test] + fn test_validate_invalid_update_payload() { + let invalid_payloads = [ + ( + "EditSite name matches LocalSite slur filter", + "slurs", + &generate_local_site( + Some(String::from("(foo|bar)")), + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_edit_site( + Some(String::from("foo site_name")), + None::, + None::, + None::, + None::, + None::, + None::, + None::, + None::, + ), + ), + ( + "EditSite name matches new slur filter", + "slurs", + &generate_local_site( + Some(String::from("(foo|bar)")), + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_edit_site( + Some(String::from("zeta site_name")), + None::, + None::, + None::, + Some(String::from("(zeta|alpha)")), + None::, + None::, + None::, + None::, + ), + ), + ( + "EditSite listing type is Subscribed, which is invalid", + "invalid_default_post_listing_type", + &generate_local_site( + None::, + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_edit_site( + Some(String::from("site_name")), + None::, + None::, + Some(ListingType::Subscribed), + None::, + None::, + None::, + None::, + None::, + ), + ), + ( + "EditSite is both private and federated", + "cant_enable_private_instance_and_federation_together", + &generate_local_site( + None::, + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_edit_site( + Some(String::from("site_name")), + None::, + None::, + None::, + None::, + Some(true), + Some(true), + None::, + None::, + ), + ), + ( + "LocalSite is private, but EditSite also makes it federated", + "cant_enable_private_instance_and_federation_together", + &generate_local_site( + None::, + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_edit_site( + Some(String::from("site_name")), + None::, + None::, + None::, + None::, + None::, + Some(true), + None::, + None::, + ), + ), + ( + "EditSite requires application, but neither it nor LocalSite has an application question", + "application_question_required", + &generate_local_site( + None::, + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_edit_site( + Some(String::from("site_name")), + None::, + None::, + None::, + None::, + None::, + None::, + None::, + Some(RegistrationMode::RequireApplication), + ), + ), + ]; + + invalid_payloads.iter().enumerate().for_each( + |( + idx, + &(reason, expected_err, local_site, edit_site), + )| { + match validate_update_payload(local_site, edit_site) { + Ok(_) => { + panic!( + "Got Ok, but validation should have failed with error: {} for reason: {}. invalid_payloads.nth({})", + expected_err, reason, idx + ) + } + Err(error) => { + assert!( + error.message.eq(&Some(String::from(expected_err))), + "Got Err {:?}, but should have failed with message: {} for reason: {}. invalid_payloads.nth({})", + error.message, + expected_err, + reason, + idx + ) + } + } + }, + ); + } + + #[test] + fn test_validate_valid_update_payload() { + let valid_payloads = [ + ( + "No changes between LocalSite and EditSite", + &generate_local_site( + None::, + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_edit_site( + None::, + None::, + None::, + None::, + None::, + None::, + None::, + None::, + None::, + ), + ), + ( + "EditSite allows clearing and changing values", + &generate_local_site( + None::, + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_edit_site( + Some(String::from("site_name")), + Some(String::new()), + Some(String::new()), + Some(ListingType::All), + Some(String::new()), + Some(false), + Some(true), + Some(String::new()), + Some(RegistrationMode::Open), + ), + ), + ( + "EditSite name passes slur filter regex", + &generate_local_site( + Some(String::from("(foo|bar)")), + true, + false, + None::, + RegistrationMode::Open, + ), + &generate_edit_site( + Some(String::from("foo site_name")), + None::, + None::, + None::, + Some(String::new()), + None::, + None::, + None::, + None::, + ), + ), + ( + "LocalSite has application question and EditSite now requires applications,", + &generate_local_site( + None::, + true, + false, + Some(String::from("question")), + RegistrationMode::Open, + ), + &generate_edit_site( + Some(String::from("site_name")), + None::, + None::, + None::, + None::, + None::, + None::, + None::, + Some(RegistrationMode::RequireApplication), + ), + ), + ]; + + valid_payloads + .iter() + .enumerate() + .for_each(|(idx, &(reason, local_site, edit_site))| { + assert!( + validate_update_payload(local_site, edit_site).is_ok(), + "Got Err, but should have got Ok for reason: {}. valid_payloads.nth({})", + reason, + idx + ); + }) + } + + fn generate_local_site( + site_slur_filter_regex: Option, + site_is_private: bool, + site_is_federated: bool, + site_application_question: Option, + site_registration_mode: RegistrationMode, + ) -> LocalSite { + LocalSite { + id: Default::default(), + site_id: Default::default(), + site_setup: true, + enable_downvotes: false, + enable_nsfw: false, + community_creation_admin_only: false, + require_email_verification: false, + application_question: site_application_question, + private_instance: site_is_private, + default_theme: String::new(), + default_post_listing_type: ListingType::All, + legal_information: None, + hide_modlog_mod_names: false, + application_email_admins: false, + slur_filter_regex: site_slur_filter_regex, + actor_name_max_length: 0, + federation_enabled: site_is_federated, + captcha_enabled: false, + captcha_difficulty: String::new(), + published: Default::default(), + updated: None, + registration_mode: site_registration_mode, + reports_email_admins: false, + } + } + + // Allow the test helper function to have too many arguments. + // It's either this or generate the entire struct each time for testing. + #[allow(clippy::too_many_arguments)] + fn generate_edit_site( + site_name: Option, + site_description: Option, + site_sidebar: Option, + site_listing_type: Option, + site_slur_filter_regex: Option, + site_is_private: Option, + site_is_federated: Option, + site_application_question: Option, + site_registration_mode: Option, + ) -> EditSite { + EditSite { + name: site_name, + sidebar: site_sidebar, + description: site_description, + icon: None, + banner: None, + enable_downvotes: None, + enable_nsfw: None, + community_creation_admin_only: None, + require_email_verification: None, + application_question: site_application_question, + private_instance: site_is_private, + default_theme: None, + default_post_listing_type: site_listing_type, + legal_information: None, + application_email_admins: None, + hide_modlog_mod_names: None, + discussion_languages: None, + slur_filter_regex: site_slur_filter_regex, + actor_name_max_length: None, + rate_limit_message: None, + rate_limit_message_per_second: None, + rate_limit_post: None, + rate_limit_post_per_second: None, + rate_limit_register: None, + rate_limit_register_per_second: None, + rate_limit_image: None, + rate_limit_image_per_second: None, + rate_limit_comment: None, + rate_limit_comment_per_second: None, + rate_limit_search: None, + rate_limit_search_per_second: None, + federation_enabled: site_is_federated, + federation_debug: None, + captcha_enabled: None, + captcha_difficulty: None, + allowed_instances: None, + blocked_instances: None, + taglines: None, + registration_mode: site_registration_mode, + reports_email_admins: None, + auth: Default::default(), + } + } +} diff --git a/crates/utils/src/utils/validation.rs b/crates/utils/src/utils/validation.rs index 621543b47..7e09c5af8 100644 --- a/crates/utils/src/utils/validation.rs +++ b/crates/utils/src/utils/validation.rs @@ -1,7 +1,7 @@ use crate::error::{LemmyError, LemmyResult}; use itertools::Itertools; use once_cell::sync::Lazy; -use regex::Regex; +use regex::{Regex, RegexBuilder}; use totp_rs::{Secret, TOTP}; use url::Url; @@ -17,9 +17,13 @@ static CLEAN_URL_PARAMS_REGEX: Lazy = Lazy::new(|| { Regex::new(r"^utm_source|utm_medium|utm_campaign|utm_term|utm_content|gclid|gclsrc|dclid|fbclid$") .expect("compile regex") }); + const BODY_MAX_LENGTH: usize = 10000; const POST_BODY_MAX_LENGTH: usize = 50000; const BIO_MAX_LENGTH: usize = 300; +const SITE_NAME_MAX_LENGTH: usize = 20; +const SITE_NAME_MIN_LENGTH: usize = 1; +const SITE_DESCRIPTION_MAX_LENGTH: usize = 150; fn has_newline(name: &str) -> bool { name.contains('\n') @@ -88,14 +92,83 @@ pub fn is_valid_body_field(body: &Option, post: bool) -> LemmyResult<()> } pub fn is_valid_bio_field(bio: &str) -> LemmyResult<()> { - let check = bio.chars().count() <= BIO_MAX_LENGTH; - if !check { - Err(LemmyError::from_message("bio_length_overflow")) + max_length_check(bio, BIO_MAX_LENGTH, String::from("bio_length_overflow")) +} + +/// Checks the site name length, the limit as defined in the DB. +pub fn site_name_length_check(name: &str) -> LemmyResult<()> { + min_max_length_check( + name, + SITE_NAME_MIN_LENGTH, + SITE_NAME_MAX_LENGTH, + String::from("site_name_required"), + String::from("site_name_length_overflow"), + ) +} + +/// Checks the site description length, the limit as defined in the DB. +pub fn site_description_length_check(description: &str) -> LemmyResult<()> { + max_length_check( + description, + SITE_DESCRIPTION_MAX_LENGTH, + String::from("site_description_length_overflow"), + ) +} + +fn max_length_check(item: &str, max_length: usize, msg: String) -> LemmyResult<()> { + if item.len() > max_length { + Err(LemmyError::from_message(&msg)) } else { Ok(()) } } +fn min_max_length_check( + item: &str, + min_length: usize, + max_length: usize, + min_msg: String, + max_msg: String, +) -> LemmyResult<()> { + if item.len() > max_length { + Err(LemmyError::from_message(&max_msg)) + } else if item.len() < min_length { + Err(LemmyError::from_message(&min_msg)) + } else { + Ok(()) + } +} + +/// Attempts to build a regex and check it for common errors before inserting into the DB. +pub fn build_and_check_regex(regex_str_opt: &Option<&str>) -> LemmyResult> { + regex_str_opt.map_or_else( + || Ok(None::), + |regex_str| { + if regex_str.is_empty() { + // If the proposed regex is empty, return as having no regex at all; this is the same + // behavior that happens downstream before the write to the database. + return Ok(None::); + } + + RegexBuilder::new(regex_str) + .case_insensitive(true) + .build() + .map_err(|e| LemmyError::from_error_message(e, "invalid_regex")) + .and_then(|regex| { + // NOTE: It is difficult to know, in the universe of user-crafted regex, which ones + // may match against any string text. To keep it simple, we'll match the regex + // against an innocuous string - a single number - which should help catch a regex + // that accidentally matches against all strings. + if regex.is_match("1") { + return Err(LemmyError::from_message("permissive_regex")); + } + + Ok(Some(regex)) + }) + }, + ) +} + pub fn clean_url_params(url: &Url) -> Url { let mut url_out = url.clone(); if url.query().is_some() { @@ -177,13 +250,20 @@ pub fn check_site_visibility_valid( mod tests { use super::build_totp_2fa; use crate::utils::validation::{ + build_and_check_regex, check_site_visibility_valid, clean_url_params, generate_totp_2fa_secret, is_valid_actor_name, + is_valid_bio_field, is_valid_display_name, is_valid_matrix_id, is_valid_post_title, + site_description_length_check, + site_name_length_check, + BIO_MAX_LENGTH, + SITE_DESCRIPTION_MAX_LENGTH, + SITE_NAME_MAX_LENGTH, }; use url::Url; @@ -252,6 +332,126 @@ mod tests { assert!(totp.is_ok()); } + #[test] + fn test_valid_site_name() { + let valid_names = [ + (0..SITE_NAME_MAX_LENGTH).map(|_| 'A').collect::(), + String::from("A"), + ]; + let invalid_names = [ + ( + &(0..SITE_NAME_MAX_LENGTH + 1) + .map(|_| 'A') + .collect::(), + "site_name_length_overflow", + ), + (&String::new(), "site_name_required"), + ]; + + valid_names.iter().for_each(|valid_name| { + assert!( + site_name_length_check(valid_name).is_ok(), + "Expected {} of length {} to be Ok.", + valid_name, + valid_name.len() + ) + }); + + invalid_names + .iter() + .for_each(|&(invalid_name, expected_err)| { + let result = site_name_length_check(invalid_name); + + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .message + .eq(&Some(String::from(expected_err))), + "Testing {}, expected error {}", + invalid_name, + expected_err + ); + }); + } + + #[test] + fn test_valid_bio() { + assert!(is_valid_bio_field(&(0..BIO_MAX_LENGTH).map(|_| 'A').collect::()).is_ok()); + + let invalid_result = + is_valid_bio_field(&(0..BIO_MAX_LENGTH + 1).map(|_| 'A').collect::()); + + assert!( + invalid_result.is_err() + && invalid_result + .unwrap_err() + .message + .eq(&Some(String::from("bio_length_overflow"))) + ); + } + + #[test] + fn test_valid_site_description() { + assert!(site_description_length_check( + &(0..SITE_DESCRIPTION_MAX_LENGTH) + .map(|_| 'A') + .collect::() + ) + .is_ok()); + + let invalid_result = site_description_length_check( + &(0..SITE_DESCRIPTION_MAX_LENGTH + 1) + .map(|_| 'A') + .collect::(), + ); + + assert!( + invalid_result.is_err() + && invalid_result + .unwrap_err() + .message + .eq(&Some(String::from("site_description_length_overflow"))) + ); + } + + #[test] + fn test_valid_slur_regex() { + let valid_regexes = [&None, &Some(""), &Some("(foo|bar)")]; + + valid_regexes.iter().for_each(|regex| { + let result = build_and_check_regex(regex); + + assert!(result.is_ok(), "Testing regex: {:?}", regex); + }); + } + + #[test] + fn test_too_permissive_slur_regex() { + let match_everything_regexes = [ + (&Some("["), "invalid_regex"), + (&Some("(foo|bar|)"), "permissive_regex"), + (&Some(".*"), "permissive_regex"), + ]; + + match_everything_regexes + .iter() + .for_each(|&(regex_str, expected_err)| { + let result = build_and_check_regex(regex_str); + + assert!(result.is_err()); + assert!( + result + .unwrap_err() + .message + .eq(&Some(String::from(expected_err))), + "Testing regex {:?}, expected error {}", + regex_str, + expected_err + ); + }); + } + #[test] fn test_check_site_visibility_valid() { assert!(check_site_visibility_valid(true, true, &None, &None).is_err()); From ad6f244b618dd7cf2aa4bd8876e378ba62b35016 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Tue, 27 Jun 2023 08:14:46 -0400 Subject: [PATCH 36/45] Version 0.18.1-rc.1 --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 22 +++++++++++----------- crates/utils/translations | 2 +- 3 files changed, 23 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 39ef0d807..f32daf5dc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2570,7 +2570,7 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lemmy_api" -version = "0.18.0" +version = "0.18.1-rc.1" dependencies = [ "actix-web", "anyhow", @@ -2594,7 +2594,7 @@ dependencies = [ [[package]] name = "lemmy_api_common" -version = "0.18.0" +version = "0.18.1-rc.1" dependencies = [ "actix-web", "anyhow", @@ -2623,7 +2623,7 @@ dependencies = [ [[package]] name = "lemmy_api_crud" -version = "0.18.0" +version = "0.18.1-rc.1" dependencies = [ "activitypub_federation", "actix-web", @@ -2644,7 +2644,7 @@ dependencies = [ [[package]] name = "lemmy_apub" -version = "0.18.0" +version = "0.18.1-rc.1" dependencies = [ "activitypub_federation", "actix-web", @@ -2682,7 +2682,7 @@ dependencies = [ [[package]] name = "lemmy_db_schema" -version = "0.18.0" +version = "0.18.1-rc.1" dependencies = [ "activitypub_federation", "async-trait", @@ -2719,7 +2719,7 @@ dependencies = [ [[package]] name = "lemmy_db_views" -version = "0.18.0" +version = "0.18.1-rc.1" dependencies = [ "diesel", "diesel-async", @@ -2736,7 +2736,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_actor" -version = "0.18.0" +version = "0.18.1-rc.1" dependencies = [ "diesel", "diesel-async", @@ -2749,7 +2749,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_moderator" -version = "0.18.0" +version = "0.18.1-rc.1" dependencies = [ "diesel", "diesel-async", @@ -2761,7 +2761,7 @@ dependencies = [ [[package]] name = "lemmy_routes" -version = "0.18.0" +version = "0.18.1-rc.1" dependencies = [ "activitypub_federation", "actix-web", @@ -2786,7 +2786,7 @@ dependencies = [ [[package]] name = "lemmy_server" -version = "0.18.0" +version = "0.18.1-rc.1" dependencies = [ "activitypub_federation", "actix-cors", @@ -2828,7 +2828,7 @@ dependencies = [ [[package]] name = "lemmy_utils" -version = "0.18.0" +version = "0.18.1-rc.1" dependencies = [ "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index 49d176b42..d01e42989 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "0.18.0" +version = "0.18.1-rc.1" edition = "2021" description = "A link aggregator for the fediverse" license = "AGPL-3.0" @@ -49,16 +49,16 @@ members = [ ] [workspace.dependencies] -lemmy_api = { version = "=0.18.0", path = "./crates/api" } -lemmy_api_crud = { version = "=0.18.0", path = "./crates/api_crud" } -lemmy_apub = { version = "=0.18.0", path = "./crates/apub" } -lemmy_utils = { version = "=0.18.0", path = "./crates/utils" } -lemmy_db_schema = { version = "=0.18.0", path = "./crates/db_schema" } -lemmy_api_common = { version = "=0.18.0", path = "./crates/api_common" } -lemmy_routes = { version = "=0.18.0", path = "./crates/routes" } -lemmy_db_views = { version = "=0.18.0", path = "./crates/db_views" } -lemmy_db_views_actor = { version = "=0.18.0", path = "./crates/db_views_actor" } -lemmy_db_views_moderator = { version = "=0.18.0", path = "./crates/db_views_moderator" } +lemmy_api = { version = "=0.18.1-rc.1", path = "./crates/api" } +lemmy_api_crud = { version = "=0.18.1-rc.1", path = "./crates/api_crud" } +lemmy_apub = { version = "=0.18.1-rc.1", path = "./crates/apub" } +lemmy_utils = { version = "=0.18.1-rc.1", path = "./crates/utils" } +lemmy_db_schema = { version = "=0.18.1-rc.1", path = "./crates/db_schema" } +lemmy_api_common = { version = "=0.18.1-rc.1", path = "./crates/api_common" } +lemmy_routes = { version = "=0.18.1-rc.1", path = "./crates/routes" } +lemmy_db_views = { version = "=0.18.1-rc.1", path = "./crates/db_views" } +lemmy_db_views_actor = { version = "=0.18.1-rc.1", path = "./crates/db_views_actor" } +lemmy_db_views_moderator = { version = "=0.18.1-rc.1", path = "./crates/db_views_moderator" } activitypub_federation = { version = "0.4.4", default-features = false, features = ["actix-web"] } diesel = "2.1.0" diesel_migrations = "2.1.0" diff --git a/crates/utils/translations b/crates/utils/translations index 7fc71d086..b3cca4b7e 160000 --- a/crates/utils/translations +++ b/crates/utils/translations @@ -1 +1 @@ -Subproject commit 7fc71d0860bbe5c6d620ec27112350ffe5b9229c +Subproject commit b3cca4b7e26dd7d9a389f75c6e50a489d93791ea From e4b739320c810e0f3eaf81c2be96dd5fc9a9a922 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Tue, 27 Jun 2023 14:59:58 +0200 Subject: [PATCH 37/45] Run cargo update as part of release script (#3369) To get newest dependency patches and get rid of yanked deps Co-authored-by: Dessalines --- scripts/release.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/release.sh b/scripts/release.sh index 2df168c43..02554d5d8 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -34,6 +34,7 @@ old_tag=$(grep version Cargo.toml | head -1 | cut -d'"' -f 2) sed -i "s/{ version = \"=$old_tag\", path/{ version = \"=$new_tag\", path/g" Cargo.toml sed -i "s/version = \"$old_tag\"/version = \"$new_tag\"/g" Cargo.toml git add Cargo.toml +cargo update cargo check git add Cargo.lock popd From bef76630c5d74805ff367bfab787d72498b6927d Mon Sep 17 00:00:00 2001 From: dullbananas Date: Wed, 28 Jun 2023 02:19:26 -0700 Subject: [PATCH 38/45] Remove redundant calls to `Iterator::collect` (#3365) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Remove redundant calls to `Iterator::collect` * Update mentions.rs * Add clippy lints and run fmt * CI ran on the wrong commit again  --- .woodpecker.yml | 3 +++ crates/api_common/src/build_response.rs | 1 - crates/api_crud/src/comment/create.rs | 2 +- crates/apub/src/activities/community/mod.rs | 5 ++--- crates/apub/src/mentions.rs | 10 +++------- crates/db_schema/src/impls/comment.rs | 3 +-- scripts/fix-clippy.sh | 5 ++++- 7 files changed, 14 insertions(+), 15 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index c2f6505c9..ec3436def 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -60,6 +60,9 @@ pipeline: -D clippy::unused_self -A clippy::uninlined_format_args -D clippy::get_first + -D clippy::explicit_into_iter_loop + -D clippy::explicit_iter_loop + -D clippy::needless_collect - cargo clippy --workspace --features console -- -D clippy::unwrap_used -D clippy::indexing_slicing diff --git a/crates/api_common/src/build_response.rs b/crates/api_common/src/build_response.rs index 374a74d91..328827b2c 100644 --- a/crates/api_common/src/build_response.rs +++ b/crates/api_common/src/build_response.rs @@ -98,7 +98,6 @@ pub async fn send_local_notifs( for mention in mentions .iter() .filter(|m| m.is_local(&context.settings().hostname) && m.name.ne(&person.name)) - .collect::>() { let mention_name = mention.name.clone(); let user_view = LocalUserView::read_from_name(context.pool(), &mention_name).await; diff --git a/crates/api_crud/src/comment/create.rs b/crates/api_crud/src/comment/create.rs index b3b1efecd..1bcc78483 100644 --- a/crates/api_crud/src/comment/create.rs +++ b/crates/api_crud/src/comment/create.rs @@ -191,7 +191,7 @@ impl PerformCrud for CreateComment { pub fn check_comment_depth(comment: &Comment) -> Result<(), LemmyError> { let path = &comment.path.0; - let length = path.split('.').collect::>().len(); + let length = path.split('.').count(); if length > MAX_COMMENT_DEPTH_LIMIT { Err(LemmyError::from_message("max_comment_depth_reached")) } else { diff --git a/crates/apub/src/activities/community/mod.rs b/crates/apub/src/activities/community/mod.rs index b770c47ac..010bad4f4 100644 --- a/crates/apub/src/activities/community/mod.rs +++ b/crates/apub/src/activities/community/mod.rs @@ -43,12 +43,11 @@ pub(crate) async fn send_activity_in_community( // send to user followers if !is_mod_action { - inboxes.append( + inboxes.extend( &mut PersonFollower::list_followers(context.pool(), actor.id) .await? .into_iter() - .map(|p| ApubPerson(p).shared_inbox_or_inbox()) - .collect(), + .map(|p| ApubPerson(p).shared_inbox_or_inbox()), ); } diff --git a/crates/apub/src/mentions.rs b/crates/apub/src/mentions.rs index 4de822cc7..088f84d0d 100644 --- a/crates/apub/src/mentions.rs +++ b/crates/apub/src/mentions.rs @@ -11,10 +11,7 @@ use lemmy_db_schema::{ traits::Crud, utils::DbPool, }; -use lemmy_utils::{ - error::LemmyError, - utils::mention::{scrape_text_for_mentions, MentionData}, -}; +use lemmy_utils::{error::LemmyError, utils::mention::scrape_text_for_mentions}; use serde::{Deserialize, Serialize}; use serde_json::Value; use url::Url; @@ -67,10 +64,9 @@ pub async fn collect_non_local_mentions( let mentions = scrape_text_for_mentions(&comment.content) .into_iter() // Filter only the non-local ones - .filter(|m| !m.is_local(&context.settings().hostname)) - .collect::>(); + .filter(|m| !m.is_local(&context.settings().hostname)); - for mention in &mentions { + for mention in mentions { let identifier = format!("{}@{}", mention.name, mention.domain); let person = webfinger_resolve_actor::(&identifier, context).await; if let Ok(person) = person { diff --git a/crates/db_schema/src/impls/comment.rs b/crates/db_schema/src/impls/comment.rs index 8aa8c1793..ea66347db 100644 --- a/crates/db_schema/src/impls/comment.rs +++ b/crates/db_schema/src/impls/comment.rs @@ -99,8 +99,7 @@ impl Comment { // left join comment c2 on c2.path <@ c.path and c2.path != c.path // group by c.id - let path_split = parent_path.0.split('.').collect::>(); - let parent_id = path_split.get(1); + let parent_id = parent_path.0.split('.').nth(1); if let Some(parent_id) = parent_id { let top_parent = format!("0.{}", parent_id); diff --git a/scripts/fix-clippy.sh b/scripts/fix-clippy.sh index 25b4b22ca..759de5773 100755 --- a/scripts/fix-clippy.sh +++ b/scripts/fix-clippy.sh @@ -14,7 +14,10 @@ cargo clippy --workspace --fix --allow-staged --allow-dirty --tests --all-target -D clippy::manual_string_new -D clippy::redundant_closure_for_method_calls \ -D clippy::unused_self \ -A clippy::uninlined_format_args \ - -D clippy::get_first + -D clippy::get_first \ + -D clippy::explicit_into_iter_loop \ + -D clippy::explicit_iter_loop \ + -D clippy::needless_collect cargo clippy --workspace --features console -- \ -D clippy::unwrap_used \ From 0f91759e4d1f7092ae23302ccb6426250a07dab2 Mon Sep 17 00:00:00 2001 From: Jan Klass Date: Wed, 28 Jun 2023 11:25:46 +0200 Subject: [PATCH 39/45] docs(api): Add api-common info on generating TypeScript bindings (#3330) --- crates/api_common/README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/crates/api_common/README.md b/crates/api_common/README.md index bf916bf0c..f6a16c53c 100644 --- a/crates/api_common/README.md +++ b/crates/api_common/README.md @@ -24,3 +24,10 @@ As you can see, each API endpoint needs a parameter type ( GetPosts), path (/pos For a real example of a Lemmy API client, look at [lemmyBB](https://github.com/LemmyNet/lemmyBB/tree/main/src/api). Lemmy also provides a websocket API. You can find the full websocket code in [this file](https://github.com/LemmyNet/lemmy/blob/main/src/api_routes_websocket.rs). + +## Generate TypeScript bindings + +TypeScript bindings (API types) can be generated by running `cargo test --features full`. +The ts files be generated into a `bindings` folder. + +This crate uses [`ts_rs`](https://docs.rs/ts-rs/6.2.1/ts_rs/#traits) macros `derive(TS)` and `ts(export)` to attribute types for binding generating. From ffc049078e834f1a74b20307014f29fca3f8459a Mon Sep 17 00:00:00 2001 From: c-andy-candies <74613851+c-andy-candies@users.noreply.github.com> Date: Wed, 28 Jun 2023 18:44:16 +0200 Subject: [PATCH 40/45] Fix missing sorting types (#3370) * Fix missing sorting types * Reordered sort_type_enum --- .../down.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/2023-06-20-191145_add_listingtype_sorttype_3_6_9_months_enums/down.sql b/migrations/2023-06-20-191145_add_listingtype_sorttype_3_6_9_months_enums/down.sql index f8493fd26..5b135223e 100644 --- a/migrations/2023-06-20-191145_add_listingtype_sorttype_3_6_9_months_enums/down.sql +++ b/migrations/2023-06-20-191145_add_listingtype_sorttype_3_6_9_months_enums/down.sql @@ -4,7 +4,7 @@ update local_user set default_sort_type = 'TopDay' where default_sort_type in (' -- rename the old enum alter type sort_type_enum rename to sort_type_enum__; -- create the new enum -CREATE TYPE sort_type_enum AS ENUM ('Active', 'Hot', 'New', 'Old', 'TopDay', 'TopWeek', 'TopMonth', 'TopYear', 'TopAll', 'MostComments', 'NewComments'); +CREATE TYPE sort_type_enum AS ENUM ('Active', 'Hot', 'New', 'Old', 'TopDay', 'TopWeek', 'TopMonth', 'TopYear', 'TopAll', 'MostComments', 'NewComments', 'TopHour', 'TopSixHour', 'TopTwelveHour'); -- alter all you enum columns alter table local_user From ec18fd986959b950acb943950d9dfffaf18dee9d Mon Sep 17 00:00:00 2001 From: Sander Saarend Date: Wed, 28 Jun 2023 19:57:49 +0300 Subject: [PATCH 41/45] Fix cargo warnings (#3397) --- crates/api_crud/Cargo.toml | 2 +- crates/db_schema/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/api_crud/Cargo.toml b/crates/api_crud/Cargo.toml index abe747b15..a4dfb4add 100644 --- a/crates/api_crud/Cargo.toml +++ b/crates/api_crud/Cargo.toml @@ -22,5 +22,5 @@ tracing = { workspace = true } url = { workspace = true } async-trait = { workspace = true } webmention = "0.4.0" -chrono = { worspace = true } +chrono = { workspace = true } uuid = { workspace = true } \ No newline at end of file diff --git a/crates/db_schema/Cargo.toml b/crates/db_schema/Cargo.toml index 69affde88..26f9b7901 100644 --- a/crates/db_schema/Cargo.toml +++ b/crates/db_schema/Cargo.toml @@ -48,7 +48,7 @@ rustls = { workspace = true } futures-util = { workspace = true } tokio-postgres = { workspace = true } tokio-postgres-rustls = { workspace = true } -uuid = { features = ["v4"] } +uuid = { workspace = true, features = ["v4"] } [dev-dependencies] serial_test = { workspace = true } From c216153dfb3cb2d08b72fa466fe522b3c7accde7 Mon Sep 17 00:00:00 2001 From: Nutomic Date: Wed, 28 Jun 2023 18:58:23 +0200 Subject: [PATCH 42/45] Update activitypub-federation crate to 0.4.5 (#3379) https://github.com/LemmyNet/activitypub-federation-rust/releases/tag/0.4.5 --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f32daf5dc..8fe0dfc49 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14,9 +14,9 @@ dependencies = [ [[package]] name = "activitypub_federation" -version = "0.4.4" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27540f6c4b72c91176610ed5279061a021387f972c7c6f42c41032b78a808267" +checksum = "4ab3ac148d9c0b4163a6d41040c17de7558a42224b9ecbd4e8f033aef6c254d9" dependencies = [ "activitystreams-kinds", "actix-web", From 0464c46d266be8ea657767b6c172576d78c316e8 Mon Sep 17 00:00:00 2001 From: Nick Shockey Date: Thu, 29 Jun 2023 04:16:11 -0400 Subject: [PATCH 43/45] Added gitattributes to normalize all files to lf (#3386) This makes it less annoying to build on windows/mac Co-authored-by: Dessalines --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 000000000..8ad74f78d --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Normalize EOL for all files that Git considers text files. +* text=auto eol=lf From 6c64cb52336c380f37e24b41a9b6a0420aa42f0d Mon Sep 17 00:00:00 2001 From: Dessalines Date: Thu, 29 Jun 2023 10:17:59 -0400 Subject: [PATCH 44/45] Fixing release script. (#3398) * Fixing release script. * Version 0.18.1-rc.2 * Removing cargo update from release script. * Fixing topdir location. --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 22 +++++++++++----------- crates/utils/translations | 2 +- scripts/release.sh | 11 ++++++----- 4 files changed, 29 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8fe0dfc49..4daec1ebf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2570,7 +2570,7 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lemmy_api" -version = "0.18.1-rc.1" +version = "0.18.1-rc.2" dependencies = [ "actix-web", "anyhow", @@ -2594,7 +2594,7 @@ dependencies = [ [[package]] name = "lemmy_api_common" -version = "0.18.1-rc.1" +version = "0.18.1-rc.2" dependencies = [ "actix-web", "anyhow", @@ -2623,7 +2623,7 @@ dependencies = [ [[package]] name = "lemmy_api_crud" -version = "0.18.1-rc.1" +version = "0.18.1-rc.2" dependencies = [ "activitypub_federation", "actix-web", @@ -2644,7 +2644,7 @@ dependencies = [ [[package]] name = "lemmy_apub" -version = "0.18.1-rc.1" +version = "0.18.1-rc.2" dependencies = [ "activitypub_federation", "actix-web", @@ -2682,7 +2682,7 @@ dependencies = [ [[package]] name = "lemmy_db_schema" -version = "0.18.1-rc.1" +version = "0.18.1-rc.2" dependencies = [ "activitypub_federation", "async-trait", @@ -2719,7 +2719,7 @@ dependencies = [ [[package]] name = "lemmy_db_views" -version = "0.18.1-rc.1" +version = "0.18.1-rc.2" dependencies = [ "diesel", "diesel-async", @@ -2736,7 +2736,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_actor" -version = "0.18.1-rc.1" +version = "0.18.1-rc.2" dependencies = [ "diesel", "diesel-async", @@ -2749,7 +2749,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_moderator" -version = "0.18.1-rc.1" +version = "0.18.1-rc.2" dependencies = [ "diesel", "diesel-async", @@ -2761,7 +2761,7 @@ dependencies = [ [[package]] name = "lemmy_routes" -version = "0.18.1-rc.1" +version = "0.18.1-rc.2" dependencies = [ "activitypub_federation", "actix-web", @@ -2786,7 +2786,7 @@ dependencies = [ [[package]] name = "lemmy_server" -version = "0.18.1-rc.1" +version = "0.18.1-rc.2" dependencies = [ "activitypub_federation", "actix-cors", @@ -2828,7 +2828,7 @@ dependencies = [ [[package]] name = "lemmy_utils" -version = "0.18.1-rc.1" +version = "0.18.1-rc.2" dependencies = [ "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index d01e42989..f8dcf81b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "0.18.1-rc.1" +version = "0.18.1-rc.2" edition = "2021" description = "A link aggregator for the fediverse" license = "AGPL-3.0" @@ -49,16 +49,16 @@ members = [ ] [workspace.dependencies] -lemmy_api = { version = "=0.18.1-rc.1", path = "./crates/api" } -lemmy_api_crud = { version = "=0.18.1-rc.1", path = "./crates/api_crud" } -lemmy_apub = { version = "=0.18.1-rc.1", path = "./crates/apub" } -lemmy_utils = { version = "=0.18.1-rc.1", path = "./crates/utils" } -lemmy_db_schema = { version = "=0.18.1-rc.1", path = "./crates/db_schema" } -lemmy_api_common = { version = "=0.18.1-rc.1", path = "./crates/api_common" } -lemmy_routes = { version = "=0.18.1-rc.1", path = "./crates/routes" } -lemmy_db_views = { version = "=0.18.1-rc.1", path = "./crates/db_views" } -lemmy_db_views_actor = { version = "=0.18.1-rc.1", path = "./crates/db_views_actor" } -lemmy_db_views_moderator = { version = "=0.18.1-rc.1", path = "./crates/db_views_moderator" } +lemmy_api = { version = "=0.18.1-rc.2", path = "./crates/api" } +lemmy_api_crud = { version = "=0.18.1-rc.2", path = "./crates/api_crud" } +lemmy_apub = { version = "=0.18.1-rc.2", path = "./crates/apub" } +lemmy_utils = { version = "=0.18.1-rc.2", path = "./crates/utils" } +lemmy_db_schema = { version = "=0.18.1-rc.2", path = "./crates/db_schema" } +lemmy_api_common = { version = "=0.18.1-rc.2", path = "./crates/api_common" } +lemmy_routes = { version = "=0.18.1-rc.2", path = "./crates/routes" } +lemmy_db_views = { version = "=0.18.1-rc.2", path = "./crates/db_views" } +lemmy_db_views_actor = { version = "=0.18.1-rc.2", path = "./crates/db_views_actor" } +lemmy_db_views_moderator = { version = "=0.18.1-rc.2", path = "./crates/db_views_moderator" } activitypub_federation = { version = "0.4.4", default-features = false, features = ["actix-web"] } diesel = "2.1.0" diesel_migrations = "2.1.0" diff --git a/crates/utils/translations b/crates/utils/translations index b3cca4b7e..5a9d44656 160000 --- a/crates/utils/translations +++ b/crates/utils/translations @@ -1 +1 @@ -Subproject commit b3cca4b7e26dd7d9a389f75c6e50a489d93791ea +Subproject commit 5a9d44656e2658ab7cb2dbec3fd1bfaf57654533 diff --git a/scripts/release.sh b/scripts/release.sh index 02554d5d8..76cb2dbfb 100755 --- a/scripts/release.sh +++ b/scripts/release.sh @@ -6,10 +6,14 @@ set -e new_tag="$1" third_semver=$(echo $new_tag | cut -d "." -f 3) +# Goto the upper route +CWD="$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)" +cd $CWD/../ + # The ansible and docker installs should only update for non release-candidates # IE, when the third semver is a number, not '2-rc' if [ ! -z "${third_semver##*[!0-9]*}" ]; then - pushd ../docker + pushd docker sed -i "s/dessalines\/lemmy:.*/dessalines\/lemmy:$new_tag/" docker-compose.yml sed -i "s/dessalines\/lemmy-ui:.*/dessalines\/lemmy-ui:$new_tag/" docker-compose.yml sed -i "s/dessalines\/lemmy-ui:.*/dessalines\/lemmy-ui:$new_tag/" federation/docker-compose.yml @@ -18,7 +22,7 @@ if [ ! -z "${third_semver##*[!0-9]*}" ]; then popd # Setting the version for Ansible - pushd ../../lemmy-ansible + pushd ../lemmy-ansible echo $new_tag > "VERSION" git add "VERSION" git commit -m"Updating VERSION" @@ -29,15 +33,12 @@ if [ ! -z "${third_semver##*[!0-9]*}" ]; then fi # Update crate versions -pushd .. old_tag=$(grep version Cargo.toml | head -1 | cut -d'"' -f 2) sed -i "s/{ version = \"=$old_tag\", path/{ version = \"=$new_tag\", path/g" Cargo.toml sed -i "s/version = \"$old_tag\"/version = \"$new_tag\"/g" Cargo.toml git add Cargo.toml -cargo update cargo check git add Cargo.lock -popd # Update the submodules git submodule update --remote From 3159eedd9928f7ae0e01eea22ab721a795687888 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Thu, 29 Jun 2023 10:45:59 -0400 Subject: [PATCH 45/45] Version 0.18.1-rc.4 --- Cargo.lock | 22 +++++++++++----------- Cargo.toml | 22 +++++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4daec1ebf..1fb6746e0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2570,7 +2570,7 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lemmy_api" -version = "0.18.1-rc.2" +version = "0.18.1-rc.4" dependencies = [ "actix-web", "anyhow", @@ -2594,7 +2594,7 @@ dependencies = [ [[package]] name = "lemmy_api_common" -version = "0.18.1-rc.2" +version = "0.18.1-rc.4" dependencies = [ "actix-web", "anyhow", @@ -2623,7 +2623,7 @@ dependencies = [ [[package]] name = "lemmy_api_crud" -version = "0.18.1-rc.2" +version = "0.18.1-rc.4" dependencies = [ "activitypub_federation", "actix-web", @@ -2644,7 +2644,7 @@ dependencies = [ [[package]] name = "lemmy_apub" -version = "0.18.1-rc.2" +version = "0.18.1-rc.4" dependencies = [ "activitypub_federation", "actix-web", @@ -2682,7 +2682,7 @@ dependencies = [ [[package]] name = "lemmy_db_schema" -version = "0.18.1-rc.2" +version = "0.18.1-rc.4" dependencies = [ "activitypub_federation", "async-trait", @@ -2719,7 +2719,7 @@ dependencies = [ [[package]] name = "lemmy_db_views" -version = "0.18.1-rc.2" +version = "0.18.1-rc.4" dependencies = [ "diesel", "diesel-async", @@ -2736,7 +2736,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_actor" -version = "0.18.1-rc.2" +version = "0.18.1-rc.4" dependencies = [ "diesel", "diesel-async", @@ -2749,7 +2749,7 @@ dependencies = [ [[package]] name = "lemmy_db_views_moderator" -version = "0.18.1-rc.2" +version = "0.18.1-rc.4" dependencies = [ "diesel", "diesel-async", @@ -2761,7 +2761,7 @@ dependencies = [ [[package]] name = "lemmy_routes" -version = "0.18.1-rc.2" +version = "0.18.1-rc.4" dependencies = [ "activitypub_federation", "actix-web", @@ -2786,7 +2786,7 @@ dependencies = [ [[package]] name = "lemmy_server" -version = "0.18.1-rc.2" +version = "0.18.1-rc.4" dependencies = [ "activitypub_federation", "actix-cors", @@ -2828,7 +2828,7 @@ dependencies = [ [[package]] name = "lemmy_utils" -version = "0.18.1-rc.2" +version = "0.18.1-rc.4" dependencies = [ "actix-web", "anyhow", diff --git a/Cargo.toml b/Cargo.toml index f8dcf81b7..6fbd91d1b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "0.18.1-rc.2" +version = "0.18.1-rc.4" edition = "2021" description = "A link aggregator for the fediverse" license = "AGPL-3.0" @@ -49,16 +49,16 @@ members = [ ] [workspace.dependencies] -lemmy_api = { version = "=0.18.1-rc.2", path = "./crates/api" } -lemmy_api_crud = { version = "=0.18.1-rc.2", path = "./crates/api_crud" } -lemmy_apub = { version = "=0.18.1-rc.2", path = "./crates/apub" } -lemmy_utils = { version = "=0.18.1-rc.2", path = "./crates/utils" } -lemmy_db_schema = { version = "=0.18.1-rc.2", path = "./crates/db_schema" } -lemmy_api_common = { version = "=0.18.1-rc.2", path = "./crates/api_common" } -lemmy_routes = { version = "=0.18.1-rc.2", path = "./crates/routes" } -lemmy_db_views = { version = "=0.18.1-rc.2", path = "./crates/db_views" } -lemmy_db_views_actor = { version = "=0.18.1-rc.2", path = "./crates/db_views_actor" } -lemmy_db_views_moderator = { version = "=0.18.1-rc.2", path = "./crates/db_views_moderator" } +lemmy_api = { version = "=0.18.1-rc.4", path = "./crates/api" } +lemmy_api_crud = { version = "=0.18.1-rc.4", path = "./crates/api_crud" } +lemmy_apub = { version = "=0.18.1-rc.4", path = "./crates/apub" } +lemmy_utils = { version = "=0.18.1-rc.4", path = "./crates/utils" } +lemmy_db_schema = { version = "=0.18.1-rc.4", path = "./crates/db_schema" } +lemmy_api_common = { version = "=0.18.1-rc.4", path = "./crates/api_common" } +lemmy_routes = { version = "=0.18.1-rc.4", path = "./crates/routes" } +lemmy_db_views = { version = "=0.18.1-rc.4", path = "./crates/db_views" } +lemmy_db_views_actor = { version = "=0.18.1-rc.4", path = "./crates/db_views_actor" } +lemmy_db_views_moderator = { version = "=0.18.1-rc.4", path = "./crates/db_views_moderator" } activitypub_federation = { version = "0.4.4", default-features = false, features = ["actix-web"] } diesel = "2.1.0" diesel_migrations = "2.1.0"